From e21a3fe0cd91dfeae76bce4d58c7611e78b60fbf Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 14 Oct 2016 23:10:07 +0200 Subject: [PATCH 1/3] Adding sync of follow relationships to Neo4J, accounts/suggestions API --- .gitignore | 1 + Gemfile | 1 + Gemfile.lock | 15 ++++++++++-- .../components/actions/suggestions.jsx | 15 ++++++++++++ app/controllers/api/v1/accounts_controller.rb | 6 ++++- app/models/follow.rb | 24 +++++++++++++++++++ app/models/follow_suggestion.rb | 7 ++++++ app/views/api/v1/accounts/suggestions.rabl | 2 ++ config/application.rb | 4 ++-- config/initializers/neography.rb | 5 ++++ config/routes.rb | 1 + 11 files changed, 76 insertions(+), 5 deletions(-) create mode 100644 app/assets/javascripts/components/actions/suggestions.jsx create mode 100644 app/models/follow_suggestion.rb create mode 100644 app/views/api/v1/accounts/suggestions.rabl create mode 100644 config/initializers/neography.rb diff --git a/.gitignore b/.gitignore index 340657ee9..5dabc59d9 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ public/assets .env .env.* node_modules/ +neo4j/ diff --git a/Gemfile b/Gemfile index f145531a3..1cab16a39 100644 --- a/Gemfile +++ b/Gemfile @@ -39,6 +39,7 @@ gem 'will_paginate' gem 'rack-attack' gem 'sidekiq' gem 'ledermann-rails-settings' +gem 'neography' gem 'react-rails' gem 'browserify-rails' diff --git a/Gemfile.lock b/Gemfile.lock index 43e36cc72..1e6068a44 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -97,6 +97,7 @@ GEM dotenv (= 2.1.1) railties (>= 4.0, < 5.1) erubis (2.7.0) + excon (0.53.0) execjs (2.7.0) fabrication (2.15.2) fast_blank (1.0.0) @@ -165,13 +166,21 @@ GEM mime-types-data (3.2016.0521) mimemagic (0.3.0) mini_portile2 (2.1.0) - minitest (5.9.0) + minitest (5.9.1) multi_json (1.12.1) + neography (1.8.0) + excon (>= 0.33.0) + json (>= 1.7.7) + multi_json (>= 1.3.2) + os (>= 0.9.6) + rake (>= 0.8.7) + rubyzip (>= 1.0.0) nio4r (1.2.1) nokogiri (1.6.8.1) mini_portile2 (~> 2.1.0) oj (2.17.3) orm_adapter (0.5.0) + os (0.9.6) ostatus2 (1.0.1) addressable (~> 2.4) http (~> 2.0) @@ -236,7 +245,7 @@ GEM rake (>= 0.8.7) thor (>= 0.18.1, < 2.0) rainbow (2.1.0) - rake (11.2.2) + rake (11.3.0) rdoc (4.2.2) json (~> 1.4) react-rails (1.8.2) @@ -281,6 +290,7 @@ GEM ruby-progressbar (~> 1.7) unicode-display_width (~> 1.0, >= 1.0.1) ruby-progressbar (1.8.1) + rubyzip (1.2.0) safe_yaml (1.0.4) sass (3.4.22) sass-rails (5.0.6) @@ -370,6 +380,7 @@ DEPENDENCIES letter_opener link_header lograge + neography nokogiri oj ostatus2 diff --git a/app/assets/javascripts/components/actions/suggestions.jsx b/app/assets/javascripts/components/actions/suggestions.jsx new file mode 100644 index 000000000..562a901c5 --- /dev/null +++ b/app/assets/javascripts/components/actions/suggestions.jsx @@ -0,0 +1,15 @@ +import api from '../api'; + +export const SUGGESTIONS_FETCH_REQUEST = 'SUGGESTIONS_FETCH_REQUEST'; +export const SUGGESTIONS_FETCH_SUCCESS = 'SUGGESTIONS_FETCH_SUCCESS'; +export const SUGGESTIONS_FETCH_FAIL = 'SUGGESTIONS_FETCH_FAIL'; + +export function fetchSuggestions() { + return (dispatch, getState) => { + api(getState).get('/api/v1/accounts/suggestions').then(response => { + console.log(response.data); + }).catch(error => { + console.error(error); + }); + }; +}; diff --git a/app/controllers/api/v1/accounts_controller.rb b/app/controllers/api/v1/accounts_controller.rb index 930f60cc3..7757fd7f8 100644 --- a/app/controllers/api/v1/accounts_controller.rb +++ b/app/controllers/api/v1/accounts_controller.rb @@ -1,6 +1,6 @@ class Api::V1::AccountsController < ApiController before_action :doorkeeper_authorize! - before_action :set_account, except: :verify_credentials + before_action :set_account, except: [:verify_credentials, :suggestions] respond_to :json def show @@ -19,6 +19,10 @@ class Api::V1::AccountsController < ApiController @followers = @account.followers end + def suggestions + @accounts = FollowSuggestion.get(current_user.account_id) + end + def statuses @statuses = @account.statuses.with_includes.with_counters.paginate_by_max_id(20, params[:max_id], params[:since_id]).to_a end diff --git a/app/models/follow.rb b/app/models/follow.rb index 4f47580e1..1c52f24c0 100644 --- a/app/models/follow.rb +++ b/app/models/follow.rb @@ -22,4 +22,28 @@ class Follow < ApplicationRecord def title destroyed? ? "#{account.acct} is no longer following #{target_account.acct}" : "#{account.acct} started following #{target_account.acct}" end + + after_create :add_to_graph + after_destroy :remove_from_graph + + private + + def add_to_graph + neo = Neography::Rest.new + + a = neo.create_unique_node('account_index', 'Account', account_id.to_s, account_id: account_id) + b = neo.create_unique_node('account_index', 'Account', target_account_id.to_s, account_id: target_account_id) + + neo.create_unique_relationship('follow_index', 'Follow', id.to_s, 'follows', a, b) + rescue Neography::NeographyError => e + Rails.logger.error e + end + + def remove_from_graph + neo = Neography::Rest.new + rel = neo.get_relationship_index('follow_index', 'Follow', id.to_s) + neo.delete_relationship(rel) + rescue Neography::NeographyError => e + Rails.logger.error e + end end diff --git a/app/models/follow_suggestion.rb b/app/models/follow_suggestion.rb new file mode 100644 index 000000000..2f291bc49 --- /dev/null +++ b/app/models/follow_suggestion.rb @@ -0,0 +1,7 @@ +class FollowSuggestion + def self.get(for_account_id) + neo = Neography::Rest.new + account_ids = neo.execute_query('START a=node:account_index(Account={id}) MATCH (a)-[:follows]->(b)-[:follows]->(c) WHERE a <> c AND NOT (a)-[:follows]->(c) RETURN DISTINCT c.account_id', id: for_account_id) + Account.where(id: account_ids['data'].first) unless account_ids.empty? + end +end diff --git a/app/views/api/v1/accounts/suggestions.rabl b/app/views/api/v1/accounts/suggestions.rabl new file mode 100644 index 000000000..f4dc121ea --- /dev/null +++ b/app/views/api/v1/accounts/suggestions.rabl @@ -0,0 +1,2 @@ +collection @accounts +extends('api/v1/accounts/show') diff --git a/config/application.rb b/config/application.rb index b1c9d9f6f..3734ce928 100644 --- a/config/application.rb +++ b/config/application.rb @@ -22,8 +22,8 @@ module Mastodon # config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}').to_s] # config.i18n.default_locale = :de - config.paths.add File.join('app', 'api'), glob: File.join('**', '*.rb') - config.autoload_paths += Dir[Rails.root.join('app', 'api', '*')] + # config.paths.add File.join('app', 'api'), glob: File.join('**', '*.rb') + # config.autoload_paths += Dir[Rails.root.join('app', 'api', '*')] config.active_job.queue_adapter = :sidekiq diff --git a/config/initializers/neography.rb b/config/initializers/neography.rb new file mode 100644 index 000000000..25d54658b --- /dev/null +++ b/config/initializers/neography.rb @@ -0,0 +1,5 @@ +Neography.configure do |config| + config.protocol = "http" + config.server = ENV['NEO4J_HOST'] || 'localhost' + config.port = ENV['NEO4J_PORT'] || 7474 +end diff --git a/config/routes.rb b/config/routes.rb index ec0309dc7..f3708938a 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -73,6 +73,7 @@ Rails.application.routes.draw do collection do get :relationships get :verify_credentials + get :suggestions end member do From 20f581f79669118a1a4052408078d53c7bbe83f2 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 15 Oct 2016 12:06:30 +0200 Subject: [PATCH 2/3] Display follow suggestions --- .../components/actions/suggestions.jsx | 26 ++++++- .../compose/components/suggestions_box.jsx | 76 +++++++++++++++++++ .../containers/suggestions_container.jsx | 9 +++ .../components/features/compose/index.jsx | 14 +++- .../components/reducers/timelines.jsx | 14 +++- .../components/selectors/index.jsx | 6 ++ app/models/follow.rb | 4 +- app/models/follow_suggestion.rb | 3 + 8 files changed, 146 insertions(+), 6 deletions(-) create mode 100644 app/assets/javascripts/components/features/compose/components/suggestions_box.jsx create mode 100644 app/assets/javascripts/components/features/compose/containers/suggestions_container.jsx diff --git a/app/assets/javascripts/components/actions/suggestions.jsx b/app/assets/javascripts/components/actions/suggestions.jsx index 562a901c5..c70a4d121 100644 --- a/app/assets/javascripts/components/actions/suggestions.jsx +++ b/app/assets/javascripts/components/actions/suggestions.jsx @@ -6,10 +6,32 @@ export const SUGGESTIONS_FETCH_FAIL = 'SUGGESTIONS_FETCH_FAIL'; export function fetchSuggestions() { return (dispatch, getState) => { + dispatch(fetchSuggestionsRequest()); + api(getState).get('/api/v1/accounts/suggestions').then(response => { - console.log(response.data); + dispatch(fetchSuggestionsSuccess(response.data)); }).catch(error => { - console.error(error); + dispatch(fetchSuggestionsFail(error)); }); }; }; + +export function fetchSuggestionsRequest() { + return { + type: SUGGESTIONS_FETCH_REQUEST + }; +}; + +export function fetchSuggestionsSuccess(suggestions) { + return { + type: SUGGESTIONS_FETCH_SUCCESS, + suggestions: suggestions + }; +}; + +export function fetchSuggestionsFail(error) { + return { + type: SUGGESTIONS_FETCH_FAIL, + error: error + }; +}; diff --git a/app/assets/javascripts/components/features/compose/components/suggestions_box.jsx b/app/assets/javascripts/components/features/compose/components/suggestions_box.jsx new file mode 100644 index 000000000..289260f12 --- /dev/null +++ b/app/assets/javascripts/components/features/compose/components/suggestions_box.jsx @@ -0,0 +1,76 @@ +import PureRenderMixin from 'react-addons-pure-render-mixin'; +import ImmutablePropTypes from 'react-immutable-proptypes'; +import Avatar from '../../../components/avatar'; +import DisplayName from '../../../components/display_name'; +import { Link } from 'react-router'; + +const outerStyle = { + marginBottom: '10px', + borderTop: '1px solid #616b86' +}; + +const headerStyle = { + fontSize: '14px', + fontWeight: '500', + display: 'block', + padding: '10px', + color: '#9baec8', + background: '#454b5e', + width: '120px', + marginTop: '-18px' +}; + +const itemStyle = { + display: 'block', + padding: '10px', + color: '#9baec8', + overflow: 'hidden', + textDecoration: 'none' +}; + +const displayNameStyle = { + display: 'block', + fontWeight: '500' +}; + +const acctStyle = { + display: 'block' +}; + +const SuggestionsBox = React.createClass({ + + propTypes: { + accounts: ImmutablePropTypes.list.isRequired + }, + + mixins: [PureRenderMixin], + + render () { + const accounts = this.props.accounts.take(2); + + return ( +
+ Who to follow + + {accounts.map(account => { + let displayName = account.get('display_name'); + + if (displayName.length === 0) { + displayName = account.get('username'); + } + + return ( + +
+ {displayName} + {account.get('acct')} + + ) + })} +
+ ); + } + +}); + +export default SuggestionsBox; diff --git a/app/assets/javascripts/components/features/compose/containers/suggestions_container.jsx b/app/assets/javascripts/components/features/compose/containers/suggestions_container.jsx new file mode 100644 index 000000000..7163cb100 --- /dev/null +++ b/app/assets/javascripts/components/features/compose/containers/suggestions_container.jsx @@ -0,0 +1,9 @@ +import { connect } from 'react-redux'; +import { getSuggestions } from '../../../selectors'; +import SuggestionsBox from '../components/suggestions_box'; + +const mapStateToProps = (state) => ({ + accounts: getSuggestions(state) +}); + +export default connect(mapStateToProps)(SuggestionsBox); diff --git a/app/assets/javascripts/components/features/compose/index.jsx b/app/assets/javascripts/components/features/compose/index.jsx index 4be938158..d76afc437 100644 --- a/app/assets/javascripts/components/features/compose/index.jsx +++ b/app/assets/javascripts/components/features/compose/index.jsx @@ -4,11 +4,22 @@ import FollowFormContainer from '../ui/containers/follow_form_container'; import UploadFormContainer from '../ui/containers/upload_form_container'; import NavigationContainer from '../ui/containers/navigation_container'; import PureRenderMixin from 'react-addons-pure-render-mixin'; +import SuggestionsContainer from './containers/suggestions_container'; +import { fetchSuggestions } from '../../actions/suggestions'; +import { connect } from 'react-redux'; const Compose = React.createClass({ + propTypes: { + dispatch: React.PropTypes.func.isRequired + }, + mixins: [PureRenderMixin], + componentDidMount () { + this.props.dispatch(fetchSuggestions()); + }, + render () { return ( @@ -18,6 +29,7 @@ const Compose = React.createClass({ + ); @@ -25,4 +37,4 @@ const Compose = React.createClass({ }); -export default Compose; +export default connect()(Compose); diff --git a/app/assets/javascripts/components/reducers/timelines.jsx b/app/assets/javascripts/components/reducers/timelines.jsx index 927ac28fd..9fb84b585 100644 --- a/app/assets/javascripts/components/reducers/timelines.jsx +++ b/app/assets/javascripts/components/reducers/timelines.jsx @@ -25,6 +25,7 @@ import { STATUS_DELETE_SUCCESS } from '../actions/statuses'; import { FOLLOW_SUBMIT_SUCCESS } from '../actions/follow'; +import { SUGGESTIONS_FETCH_SUCCESS } from '../actions/suggestions'; import Immutable from 'immutable'; const initialState = Immutable.Map({ @@ -37,7 +38,8 @@ const initialState = Immutable.Map({ me: null, ancestors: Immutable.Map(), descendants: Immutable.Map(), - relationships: Immutable.Map() + relationships: Immutable.Map(), + suggestions: Immutable.List([]) }); function normalizeStatus(state, status) { @@ -189,6 +191,14 @@ function normalizeContext(state, status, ancestors, descendants) { }); }; +function normalizeSuggestions(state, accounts) { + accounts.forEach(account => { + state = state.setIn(['accounts', account.get('id')], account); + }); + + return state.set('suggestions', accounts.map(account => account.get('id'))); +}; + export default function timelines(state = initialState, action) { switch(action.type) { case TIMELINE_REFRESH_SUCCESS: @@ -221,6 +231,8 @@ export default function timelines(state = initialState, action) { return normalizeAccountTimeline(state, action.id, Immutable.fromJS(action.statuses)); case ACCOUNT_TIMELINE_EXPAND_SUCCESS: return appendNormalizedAccountTimeline(state, action.id, Immutable.fromJS(action.statuses)); + case SUGGESTIONS_FETCH_SUCCESS: + return normalizeSuggestions(state, Immutable.fromJS(action.suggestions)); default: return state; } diff --git a/app/assets/javascripts/components/selectors/index.jsx b/app/assets/javascripts/components/selectors/index.jsx index c1317b38e..c3c007f28 100644 --- a/app/assets/javascripts/components/selectors/index.jsx +++ b/app/assets/javascripts/components/selectors/index.jsx @@ -79,3 +79,9 @@ export const getNotifications = createSelector([getNotificationsBase], (base) => return arr; }); + +const getSuggestionsBase = (state) => state.getIn(['timelines', 'suggestions']); + +export const getSuggestions = createSelector([getSuggestionsBase, getAccounts], (base, accounts) => { + return base.map(accountId => accounts.get(accountId)); +}); diff --git a/app/models/follow.rb b/app/models/follow.rb index 1c52f24c0..95b6bd146 100644 --- a/app/models/follow.rb +++ b/app/models/follow.rb @@ -35,7 +35,7 @@ class Follow < ApplicationRecord b = neo.create_unique_node('account_index', 'Account', target_account_id.to_s, account_id: target_account_id) neo.create_unique_relationship('follow_index', 'Follow', id.to_s, 'follows', a, b) - rescue Neography::NeographyError => e + rescue Neography::NeographyError, Excon::Error::Socket => e Rails.logger.error e end @@ -43,7 +43,7 @@ class Follow < ApplicationRecord neo = Neography::Rest.new rel = neo.get_relationship_index('follow_index', 'Follow', id.to_s) neo.delete_relationship(rel) - rescue Neography::NeographyError => e + rescue Neography::NeographyError, Excon::Error::Socket => e Rails.logger.error e end end diff --git a/app/models/follow_suggestion.rb b/app/models/follow_suggestion.rb index 2f291bc49..f4515700a 100644 --- a/app/models/follow_suggestion.rb +++ b/app/models/follow_suggestion.rb @@ -3,5 +3,8 @@ class FollowSuggestion neo = Neography::Rest.new account_ids = neo.execute_query('START a=node:account_index(Account={id}) MATCH (a)-[:follows]->(b)-[:follows]->(c) WHERE a <> c AND NOT (a)-[:follows]->(c) RETURN DISTINCT c.account_id', id: for_account_id) Account.where(id: account_ids['data'].first) unless account_ids.empty? + rescue Neography::NeographyError, Excon::Error::Socket => e + Rails.logger.error e + [] end end From 4d336ceface783c255e62220cfa76812630ff1a1 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 15 Oct 2016 12:37:43 +0200 Subject: [PATCH 3/3] Add sync command for neo4j, fix configuration, add neo4j to docker-compose, fix seed --- .dockerignore | 1 + .env.production.sample | 2 ++ app/models/follow.rb | 4 ++++ db/seeds.rb | 2 +- docker-compose.yml | 6 ++++++ lib/tasks/mastodon.rake | 7 +++++++ 6 files changed, 21 insertions(+), 1 deletion(-) diff --git a/.dockerignore b/.dockerignore index fad29fc26..7892e503c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,3 +4,4 @@ public/system public/assets node_modules storybook +neo4j diff --git a/.env.production.sample b/.env.production.sample index 070aa0c3a..b68ba523c 100644 --- a/.env.production.sample +++ b/.env.production.sample @@ -6,6 +6,8 @@ DB_USER=postgres DB_NAME=postgres DB_PASS= DB_PORT=5432 +NEO4J_HOST=neo4j +NEO4J_PORT=7474 # Federation LOCAL_DOMAIN=example.com diff --git a/app/models/follow.rb b/app/models/follow.rb index 95b6bd146..656b28d35 100644 --- a/app/models/follow.rb +++ b/app/models/follow.rb @@ -26,6 +26,10 @@ class Follow < ApplicationRecord after_create :add_to_graph after_destroy :remove_from_graph + def sync! + add_to_graph + end + private def add_to_graph diff --git a/db/seeds.rb b/db/seeds.rb index c2bf6a16e..7e8ee8e44 100644 --- a/db/seeds.rb +++ b/db/seeds.rb @@ -1,2 +1,2 @@ web_app = Doorkeeper::Application.new(name: 'Web', superapp: true, redirect_uri: Doorkeeper.configuration.native_redirect_uri) -web_app.save(validate: false) +web_app.save! diff --git a/docker-compose.yml b/docker-compose.yml index b3b233713..18af16b60 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,6 +4,10 @@ services: image: postgres redis: image: redis + neo4j: + image: neo4j + environment: + - NEO4J_AUTH=none app: build: . env_file: .env.production @@ -15,6 +19,7 @@ services: depends_on: - db - redis + - neo4j volumes: - ./public/assets:/mastodon/public/assets - ./public/system:/mastodon/public/system @@ -24,5 +29,6 @@ services: depends_on: - db - redis + - neo4j volumes: - ./public/system:/mastodon/public/system diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake index 29e4494c4..aa07c0738 100644 --- a/lib/tasks/mastodon.rake +++ b/lib/tasks/mastodon.rake @@ -41,4 +41,11 @@ namespace :mastodon do $redis.keys('feed:*').each { |key| $redis.del(key) } end end + + namespace :graphs do + desc 'Syncs all follow relationships to Neo4J' + task sync: :environment do + Follow.find_each(&:sync!) + end + end end