Add logging and filtering to the node.js streaming API

This commit is contained in:
Eugen Rochko 2017-02-02 13:56:14 +01:00
parent 0cfd3188b4
commit 1ee4a17f37
4 changed files with 102 additions and 10 deletions

View file

@ -3,7 +3,6 @@ import NotificationsContainer from './containers/notifications_container';
import PureRenderMixin from 'react-addons-pure-render-mixin'; import PureRenderMixin from 'react-addons-pure-render-mixin';
import LoadingBarContainer from './containers/loading_bar_container'; import LoadingBarContainer from './containers/loading_bar_container';
import HomeTimeline from '../home_timeline'; import HomeTimeline from '../home_timeline';
import MentionsTimeline from '../mentions_timeline';
import Compose from '../compose'; import Compose from '../compose';
import TabsBar from './components/tabs_bar'; import TabsBar from './components/tabs_bar';
import ModalContainer from './containers/modal_container'; import ModalContainer from './containers/modal_container';

View file

@ -25,6 +25,7 @@
"emojione": "latest", "emojione": "latest",
"enzyme": "^2.4.1", "enzyme": "^2.4.1",
"es6-promise": "^3.2.1", "es6-promise": "^3.2.1",
"eventsource": "^0.2.1",
"express": "^4.14.1", "express": "^4.14.1",
"http-link-header": "^0.5.0", "http-link-header": "^0.5.0",
"immutable": "^3.8.1", "immutable": "^3.8.1",
@ -32,6 +33,7 @@
"jsdom": "^9.6.0", "jsdom": "^9.6.0",
"mocha": "^3.1.1", "mocha": "^3.1.1",
"node-sass": "^4.0.0", "node-sass": "^4.0.0",
"npmlog": "^4.0.2",
"pg": "^6.1.2", "pg": "^6.1.2",
"react": "^15.3.2", "react": "^15.3.2",
"react-addons-perf": "^15.3.2", "react-addons-perf": "^15.3.2",

View file

@ -2,6 +2,7 @@ import dotenv from 'dotenv'
import express from 'express' import express from 'express'
import redis from 'redis' import redis from 'redis'
import pg from 'pg' import pg from 'pg'
import log from 'npmlog'
dotenv.config() dotenv.config()
@ -40,6 +41,7 @@ const authenticationMiddleware = (req, res, next) => {
pgPool.connect((err, client, done) => { pgPool.connect((err, client, done) => {
if (err) { if (err) {
log.error(err)
return next(err) return next(err)
} }
@ -47,6 +49,7 @@ const authenticationMiddleware = (req, res, next) => {
done() done()
if (err) { if (err) {
log.error(err)
return next(err) return next(err)
} }
@ -66,10 +69,12 @@ const authenticationMiddleware = (req, res, next) => {
const errorMiddleware = (err, req, res, next) => { const errorMiddleware = (err, req, res, next) => {
res.writeHead(err.statusCode || 500, { 'Content-Type': 'application/json' }) res.writeHead(err.statusCode || 500, { 'Content-Type': 'application/json' })
res.end(JSON.stringify({ error: `${err}` })) res.end(JSON.stringify({ error: err.statusCode ? `${err}` : 'An unexpected error occured' }))
} }
const streamFrom = (id, res) => { const streamFrom = (id, req, res, needsFiltering = false) => {
log.verbose(`Starting stream from ${id} for ${req.accountId}`)
res.setHeader('Content-Type', 'text/event-stream') res.setHeader('Content-Type', 'text/event-stream')
res.setHeader('Transfer-Encoding', 'chunked') res.setHeader('Transfer-Encoding', 'chunked')
@ -78,11 +83,40 @@ const streamFrom = (id, res) => {
redisClient.on('message', (channel, message) => { redisClient.on('message', (channel, message) => {
const { event, payload } = JSON.parse(message) const { event, payload } = JSON.parse(message)
if (needsFiltering) {
pgPool.connect((err, client, done) => {
if (err) {
log.error(err)
return
}
const unpackedPayload = JSON.parse(payload)
const targetAccountIds = [unpackedPayload.account.id] + unpackedPayload.mentions.map(item => item.id) + (unpackedPayload.reblog ? unpackedPayload.reblog.account.id : [])
client.query('SELECT target_account_id FROM blocks WHERE account_id = $1 AND target_account_id IN ($2)', [req.accountId, targetAccountIds], (err, result) => {
done()
if (err) {
log.error(err)
return
}
if (result.rows.length > 0) {
return
}
res.write(`event: ${event}\n`) res.write(`event: ${event}\n`)
res.write(`data: ${payload}\n\n`) res.write(`data: ${payload}\n\n`)
}) })
})
} else {
res.write(`event: ${event}\n`)
res.write(`data: ${payload}\n\n`)
}
})
setInterval(() => res.write('\n'), 15000) // Heartbeat to keep connection alive
setInterval(() => res.write(':thump\n'), 15000)
redisClient.subscribe(id) redisClient.subscribe(id)
} }
@ -90,8 +124,11 @@ const streamFrom = (id, res) => {
app.use(authenticationMiddleware) app.use(authenticationMiddleware)
app.use(errorMiddleware) app.use(errorMiddleware)
app.get('/api/v1/streaming/user', (req, res) => streamFrom(`timeline:${req.accountId}`, res)) app.get('/api/v1/streaming/user', (req, res) => streamFrom(`timeline:${req.accountId}`, req, res))
app.get('/api/v1/streaming/public', (_, res) => streamFrom('timeline:public', res)) app.get('/api/v1/streaming/public', (req, res) => streamFrom('timeline:public', req, res, true))
app.get('/api/v1/streaming/hashtag', (req, res) => streamFrom(`timeline:hashtag:${req.params.tag}`, res)) app.get('/api/v1/streaming/hashtag', (req, res) => streamFrom(`timeline:hashtag:${req.params.tag}`, req, res, true))
app.listen(4000) log.level = 'verbose'
log.info(`Starting HTTP server on port ${process.env.PORT || 4000}`)
app.listen(process.env.PORT || 4000)

View file

@ -2313,6 +2313,12 @@ events@^1.0.0, events@^1.1.1, events@~1.1.0:
version "1.1.1" version "1.1.1"
resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924" resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924"
eventsource@^0.2.1:
version "0.2.1"
resolved "https://registry.yarnpkg.com/eventsource/-/eventsource-0.2.1.tgz#662bf85f376e73b5c34c2ee17db566b8419a6232"
dependencies:
original "^1.0.0"
evp_bytestokey@^1.0.0: evp_bytestokey@^1.0.0:
version "1.0.0" version "1.0.0"
resolved "https://registry.yarnpkg.com/evp_bytestokey/-/evp_bytestokey-1.0.0.tgz#497b66ad9fef65cd7c08a6180824ba1476b66e53" resolved "https://registry.yarnpkg.com/evp_bytestokey/-/evp_bytestokey-1.0.0.tgz#497b66ad9fef65cd7c08a6180824ba1476b66e53"
@ -2554,6 +2560,20 @@ gauge@~2.6.0:
strip-ansi "^3.0.1" strip-ansi "^3.0.1"
wide-align "^1.1.0" wide-align "^1.1.0"
gauge@~2.7.1:
version "2.7.2"
resolved "https://registry.yarnpkg.com/gauge/-/gauge-2.7.2.tgz#15cecc31b02d05345a5d6b0e171cdb3ad2307774"
dependencies:
aproba "^1.0.3"
console-control-strings "^1.0.0"
has-unicode "^2.0.0"
object-assign "^4.1.0"
signal-exit "^3.0.0"
string-width "^1.0.1"
strip-ansi "^3.0.1"
supports-color "^0.2.0"
wide-align "^1.1.0"
gaze@^1.0.0: gaze@^1.0.0:
version "1.1.2" version "1.1.2"
resolved "https://registry.yarnpkg.com/gaze/-/gaze-1.1.2.tgz#847224677adb8870d679257ed3388fdb61e40105" resolved "https://registry.yarnpkg.com/gaze/-/gaze-1.1.2.tgz#847224677adb8870d679257ed3388fdb61e40105"
@ -3797,6 +3817,15 @@ npmlog@4.x, npmlog@^4.0.0:
gauge "~2.6.0" gauge "~2.6.0"
set-blocking "~2.0.0" set-blocking "~2.0.0"
npmlog@^4.0.2:
version "4.0.2"
resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-4.0.2.tgz#d03950e0e78ce1527ba26d2a7592e9348ac3e75f"
dependencies:
are-we-there-yet "~1.1.2"
console-control-strings "~1.1.0"
gauge "~2.7.1"
set-blocking "~2.0.0"
nth-check@~1.0.1: nth-check@~1.0.1:
version "1.0.1" version "1.0.1"
resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-1.0.1.tgz#9929acdf628fc2c41098deab82ac580cf149aae4" resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-1.0.1.tgz#9929acdf628fc2c41098deab82ac580cf149aae4"
@ -3911,6 +3940,12 @@ optionator@^0.8.1:
type-check "~0.3.2" type-check "~0.3.2"
wordwrap "~1.0.0" wordwrap "~1.0.0"
original@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/original/-/original-1.0.0.tgz#9147f93fa1696d04be61e01bd50baeaca656bd3b"
dependencies:
url-parse "1.0.x"
os-browserify@^0.2.0: os-browserify@^0.2.0:
version "0.2.1" version "0.2.1"
resolved "https://registry.yarnpkg.com/os-browserify/-/os-browserify-0.2.1.tgz#63fc4ccee5d2d7763d26bbf8601078e6c2e0044f" resolved "https://registry.yarnpkg.com/os-browserify/-/os-browserify-0.2.1.tgz#63fc4ccee5d2d7763d26bbf8601078e6c2e0044f"
@ -4462,6 +4497,10 @@ querystring@0.2.0, querystring@^0.2.0:
version "0.2.0" version "0.2.0"
resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620"
querystringify@0.0.x:
version "0.0.4"
resolved "https://registry.yarnpkg.com/querystringify/-/querystringify-0.0.4.tgz#0cf7f84f9463ff0ae51c4c4b142d95be37724d9c"
raf@^3.1.0: raf@^3.1.0:
version "3.3.0" version "3.3.0"
resolved "https://registry.yarnpkg.com/raf/-/raf-3.3.0.tgz#93845eeffc773f8129039f677f80a36044eee2c3" resolved "https://registry.yarnpkg.com/raf/-/raf-3.3.0.tgz#93845eeffc773f8129039f677f80a36044eee2c3"
@ -4937,6 +4976,10 @@ require-main-filename@^1.0.1:
version "1.0.1" version "1.0.1"
resolved "https://registry.yarnpkg.com/require-main-filename/-/require-main-filename-1.0.1.tgz#97f717b69d48784f5f526a6c5aa8ffdda055a4d1" resolved "https://registry.yarnpkg.com/require-main-filename/-/require-main-filename-1.0.1.tgz#97f717b69d48784f5f526a6c5aa8ffdda055a4d1"
requires-port@1.0.x:
version "1.0.0"
resolved "https://registry.yarnpkg.com/requires-port/-/requires-port-1.0.0.tgz#925d2601d39ac485e091cf0da5c6e694dc3dcaff"
reselect@^2.5.4: reselect@^2.5.4:
version "2.5.4" version "2.5.4"
resolved "https://registry.yarnpkg.com/reselect/-/reselect-2.5.4.tgz#b7d23fdf00b83fa7ad0279546f8dbbbd765c7047" resolved "https://registry.yarnpkg.com/reselect/-/reselect-2.5.4.tgz#b7d23fdf00b83fa7ad0279546f8dbbbd765c7047"
@ -5349,6 +5392,10 @@ supports-color@3.1.2, supports-color@^3.1.0, supports-color@^3.1.2:
dependencies: dependencies:
has-flag "^1.0.0" has-flag "^1.0.0"
supports-color@^0.2.0:
version "0.2.0"
resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-0.2.0.tgz#d92de2694eb3f67323973d7ae3d8b55b4c22190a"
supports-color@^2.0.0: supports-color@^2.0.0:
version "2.0.0" version "2.0.0"
resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-2.0.0.tgz#535d045ce6b6363fa40117084629995e9df324c7" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-2.0.0.tgz#535d045ce6b6363fa40117084629995e9df324c7"
@ -5538,6 +5585,13 @@ url-loader@^0.5.7:
loader-utils "0.2.x" loader-utils "0.2.x"
mime "1.2.x" mime "1.2.x"
url-parse@1.0.x:
version "1.0.5"
resolved "https://registry.yarnpkg.com/url-parse/-/url-parse-1.0.5.tgz#0854860422afdcfefeb6c965c662d4800169927b"
dependencies:
querystringify "0.0.x"
requires-port "1.0.x"
url@^0.11.0, url@~0.11.0: url@^0.11.0, url@~0.11.0:
version "0.11.0" version "0.11.0"
resolved "https://registry.yarnpkg.com/url/-/url-0.11.0.tgz#3838e97cfc60521eb73c525a8e55bfdd9e2e28f1" resolved "https://registry.yarnpkg.com/url/-/url-0.11.0.tgz#3838e97cfc60521eb73c525a8e55bfdd9e2e28f1"