chinwag/chinwagsocial
chinwag/chinwagsocial
2
0
Fork 0
Code Issues Pull requests Releases 4 Wiki Activity

Reduce number of items in feeds, optimize regeneration worker slightly,

Browse source 
make regeneration worker unique, (only schedule/execute once at a time)
This commit is contained in:
Eugen Rochko 2017-04-04 13:58:34 +02:00
parent b1f3499c38
commit 82aaedec46
5 changed files with 20 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Gemfile
View file

@ -46,6 +46,8 @@ gem 'will_paginate'
gem 'rack-attack' gem 'rack-attack'
gem 'rack-cors', require: 'rack/cors' gem 'rack-cors', require: 'rack/cors'
gem 'sidekiq' gem 'sidekiq'
gem 'sidekiq-unique-jobs'
gem 'sidekiq-merger'
gem 'rails-settings-cached' gem 'rails-settings-cached'
gem 'simple-navigation' gem 'simple-navigation'
gem 'statsd-instrument' gem 'statsd-instrument'

9
Gemfile.lock
View file

@ -387,6 +387,13 @@ GEM
connection_pool (~> 2.2, >= 2.2.0) connection_pool (~> 2.2, >= 2.2.0)
rack-protection (>= 1.5.0) rack-protection (>= 1.5.0)
redis (~> 3.2, >= 3.2.1) redis (~> 3.2, >= 3.2.1)
sidekiq-merger (0.0.11)
activesupport (>= 3.2, < 6)
concurrent-ruby (~> 1.0)
sidekiq (>= 3.4, < 5)
sidekiq-unique-jobs (4.0.18)
sidekiq (>= 2.6)
thor
simple-navigation (4.0.3) simple-navigation (4.0.3)
activesupport (>= 2.3.2) activesupport (>= 2.3.2)
simple_form (3.2.1) simple_form (3.2.1)
@ -510,6 +517,8 @@ DEPENDENCIES
sass-rails (~> 5.0) sass-rails (~> 5.0)
sdoc (~> 0.4.0) sdoc (~> 0.4.0)
sidekiq sidekiq
sidekiq-merger
sidekiq-unique-jobs
simple-navigation simple-navigation
simple_form simple_form
simplecov simplecov

6
app/lib/feed_manager.rb
View file

@ -5,7 +5,7 @@ require 'singleton'
class FeedManager class FeedManager
include Singleton include Singleton
MAX_ITEMS = 800 MAX_ITEMS = 400
def key(type, id) def key(type, id)
"feed:#{type}:#{id}" "feed:#{type}:#{id}"
@ -50,9 +50,9 @@ class FeedManager
def merge_into_timeline(from_account, into_account) def merge_into_timeline(from_account, into_account)
timeline_key = key(:home, into_account.id) timeline_key = key(:home, into_account.id)
query = from_account.statuses.limit(MAX_ITEMS) query = from_account.statuses.limit(FeedManager::MAX_ITEMS / 4)
if redis.zcard(timeline_key) >= FeedManager::MAX_ITEMS if redis.zcard(timeline_key) >= FeedManager::MAX_ITEMS / 4
oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true)&.first&.last&.to_i || 0 oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true)&.first&.last&.to_i || 0
query = query.where('id > ?', oldest_home_score) query = query.where('id > ?', oldest_home_score)
end end

4
app/services/precompute_feed_service.rb
View file

@ -5,11 +5,13 @@ class PrecomputeFeedService < BaseService
# @param [Symbol] type :home or :mentions # @param [Symbol] type :home or :mentions
# @param [Account] account # @param [Account] account
def call(_, account) def call(_, account)
Status.as_home_timeline(account).limit(FeedManager::MAX_ITEMS).each do |status| redis.pipelined do
Status.as_home_timeline(account).limit(FeedManager::MAX_ITEMS / 4).each do |status|
next if status.direct_visibility? || FeedManager.instance.filter?(:home, status, account) next if status.direct_visibility? || FeedManager.instance.filter?(:home, status, account)
redis.zadd(FeedManager.instance.key(:home, account.id), status.id, status.reblog? ? status.reblog_of_id : status.id) redis.zadd(FeedManager.instance.key(:home, account.id), status.id, status.reblog? ? status.reblog_of_id : status.id)
end end
end end
end
private private

2
app/workers/regeneration_worker.rb
View file

@ -3,7 +3,7 @@
class RegenerationWorker class RegenerationWorker
include Sidekiq::Worker include Sidekiq::Worker
sidekiq_options queue: 'pull', backtrace: true sidekiq_options queue: 'pull', backtrace: true, unique: :until_executed
def perform(account_id, _ = :home) def perform(account_id, _ = :home)
PrecomputeFeedService.new.call(:home, Account.find(account_id)) PrecomputeFeedService.new.call(:home, Account.find(account_id))