Fix #4149, fix #1199 - Store emojis as unicode (#4189)

- Use unicode when selecting emoji through picker
- Convert shortcodes to unicode when storing text input server-side
- Do not convert shortcodes in JS anymore
This commit is contained in:
Eugen Rochko 2017-07-14 19:47:53 +02:00 committed by GitHub
parent c42092ba7a
commit e2685ccc81
15 changed files with 69 additions and 71 deletions

View file

@ -28,6 +28,7 @@ gem 'devise', '~> 4.2'
gem 'devise-two-factor', '~> 3.0' gem 'devise-two-factor', '~> 3.0'
gem 'doorkeeper', '~> 4.2' gem 'doorkeeper', '~> 4.2'
gem 'fast_blank', '~> 1.0' gem 'fast_blank', '~> 1.0'
gem 'gemoji', '~> 3.0'
gem 'goldfinger', '~> 1.2' gem 'goldfinger', '~> 1.2'
gem 'hiredis', '~> 0.6' gem 'hiredis', '~> 0.6'
gem 'redis-namespace', '~> 1.5' gem 'redis-namespace', '~> 1.5'

View file

@ -106,9 +106,9 @@ GEM
rack (>= 1.0.0) rack (>= 1.0.0)
rack-test (>= 0.5.4) rack-test (>= 0.5.4)
xpath (~> 2.0) xpath (~> 2.0)
charlock_holmes (0.7.3)
case_transform (0.2) case_transform (0.2)
activesupport activesupport
charlock_holmes (0.7.3)
chunky_png (1.3.8) chunky_png (1.3.8)
cld3 (3.1.3) cld3 (3.1.3)
ffi (>= 1.1.0, < 1.10.0) ffi (>= 1.1.0, < 1.10.0)
@ -163,6 +163,7 @@ GEM
fuubar (2.2.0) fuubar (2.2.0)
rspec-core (~> 3.0) rspec-core (~> 3.0)
ruby-progressbar (~> 1.4) ruby-progressbar (~> 1.4)
gemoji (3.0.0)
globalid (0.4.0) globalid (0.4.0)
activesupport (>= 4.2.0) activesupport (>= 4.2.0)
goldfinger (1.2.0) goldfinger (1.2.0)
@ -518,6 +519,7 @@ DEPENDENCIES
faker (~> 1.7) faker (~> 1.7)
fast_blank (~> 1.0) fast_blank (~> 1.0)
fuubar (~> 2.2) fuubar (~> 2.2)
gemoji (~> 3.0)
goldfinger (~> 1.2) goldfinger (~> 1.2)
hamlit-rails (~> 0.2) hamlit-rails (~> 0.2)
hiredis (~> 0.6) hiredis (~> 0.6)

View file

@ -0,0 +1,19 @@
# frozen_string_literal: true
module EmojiHelper
EMOJI_PATTERN = /(?<=[^[:alnum:]:]|\n|^):([\w+-]+):(?=[^[:alnum:]:]|$)/x
def emojify(text)
return text if text.blank?
text.gsub(EMOJI_PATTERN) do |match|
emoji = Emoji.find_by_alias($1) # rubocop:disable Rails/DynamicFindBy,Style/PerlBackrefs
if emoji
emoji.raw
else
match
end
end
end
end

View file

@ -2,8 +2,6 @@ import api from '../api';
import { updateTimeline } from './timelines'; import { updateTimeline } from './timelines';
import * as emojione from 'emojione';
export const COMPOSE_CHANGE = 'COMPOSE_CHANGE'; export const COMPOSE_CHANGE = 'COMPOSE_CHANGE';
export const COMPOSE_SUBMIT_REQUEST = 'COMPOSE_SUBMIT_REQUEST'; export const COMPOSE_SUBMIT_REQUEST = 'COMPOSE_SUBMIT_REQUEST';
export const COMPOSE_SUBMIT_SUCCESS = 'COMPOSE_SUBMIT_SUCCESS'; export const COMPOSE_SUBMIT_SUCCESS = 'COMPOSE_SUBMIT_SUCCESS';
@ -73,11 +71,14 @@ export function mentionCompose(account, router) {
export function submitCompose() { export function submitCompose() {
return function (dispatch, getState) { return function (dispatch, getState) {
const status = emojione.shortnameToUnicode(getState().getIn(['compose', 'text'], '')); const status = getState().getIn(['compose', 'text'], '');
if (!status || !status.length) { if (!status || !status.length) {
return; return;
} }
dispatch(submitComposeRequest()); dispatch(submitComposeRequest());
api(getState).post('/api/v1/statuses', { api(getState).post('/api/v1/statuses', {
status, status,
in_reply_to_id: getState().getIn(['compose', 'in_reply_to'], null), in_reply_to_id: getState().getIn(['compose', 'in_reply_to'], null),

View file

@ -6,36 +6,18 @@ const trie = new Trie(Object.keys(emojione.jsEscapeMap));
function emojify(str) { function emojify(str) {
// This walks through the string from start to end, ignoring any tags (<p>, <br>, etc.) // This walks through the string from start to end, ignoring any tags (<p>, <br>, etc.)
// and replacing valid shortnames like :smile: and :wink: as well as unicode strings // and replacing valid unicode strings
// that _aren't_ within tags with an <img> version. // that _aren't_ within tags with an <img> version.
// The goal is to be the same as an emojione.regShortNames/regUnicode replacement, but faster. // The goal is to be the same as an emojione.regUnicode replacement, but faster.
let i = -1; let i = -1;
let insideTag = false; let insideTag = false;
let insideShortname = false;
let shortnameStartIndex = -1;
let match; let match;
while (++i < str.length) { while (++i < str.length) {
const char = str.charAt(i); const char = str.charAt(i);
if (insideShortname && char === ':') { if (insideTag && char === '>') {
const shortname = str.substring(shortnameStartIndex, i + 1);
if (shortname in emojione.emojioneList) {
const unicode = emojione.emojioneList[shortname].unicode[emojione.emojioneList[shortname].unicode.length - 1];
const alt = emojione.convert(unicode.toUpperCase());
const replacement = `<img draggable="false" class="emojione" alt="${alt}" title="${shortname}" src="/emoji/${unicode}.svg" />`;
str = str.substring(0, shortnameStartIndex) + replacement + str.substring(i + 1);
i += (replacement.length - shortname.length - 1); // jump ahead the length we've added to the string
} else {
i--; // stray colon, try again
}
insideShortname = false;
} else if (insideTag && char === '>') {
insideTag = false; insideTag = false;
} else if (char === '<') { } else if (char === '<') {
insideTag = true; insideTag = true;
insideShortname = false;
} else if (!insideTag && char === ':') {
insideShortname = true;
shortnameStartIndex = i;
} else if (!insideTag && (match = trie.search(str.substring(i)))) { } else if (!insideTag && (match = trie.search(str.substring(i)))) {
const unicodeStr = match; const unicodeStr = match;
if (unicodeStr in emojione.jsEscapeMap) { if (unicodeStr in emojione.jsEscapeMap) {

View file

@ -136,7 +136,8 @@ export default class ComposeForm extends ImmutablePureComponent {
handleEmojiPick = (data) => { handleEmojiPick = (data) => {
const position = this.autosuggestTextarea.textarea.selectionStart; const position = this.autosuggestTextarea.textarea.selectionStart;
this._restoreCaret = position + data.shortname.length + 1; const emojiChar = String.fromCodePoint(parseInt(data.unicode, 16));
this._restoreCaret = position + emojiChar.length + 1;
this.props.onPickEmoji(position, data); this.props.onPickEmoji(position, data);
} }

View file

@ -109,11 +109,12 @@ export default class EmojiPickerDropdown extends React.PureComponent {
<Dropdown ref={this.setRef} className='emoji-picker__dropdown' onShow={this.onShowDropdown} onHide={this.onHideDropdown}> <Dropdown ref={this.setRef} className='emoji-picker__dropdown' onShow={this.onShowDropdown} onHide={this.onHideDropdown}>
<DropdownTrigger className='emoji-button' title={intl.formatMessage(messages.emoji)}> <DropdownTrigger className='emoji-button' title={intl.formatMessage(messages.emoji)}>
<img <img
draggable='false'
className={`emojione ${active && loading ? 'pulse-loading' : ''}`} className={`emojione ${active && loading ? 'pulse-loading' : ''}`}
alt='🙂' src='/emoji/1f602.svg' alt='🙂'
src='/emoji/1f602.svg'
/> />
</DropdownTrigger> </DropdownTrigger>
<DropdownContent className='dropdown__left'> <DropdownContent className='dropdown__left'>
{ {
this.state.active && !this.state.loading && this.state.active && !this.state.loading &&

View file

@ -118,7 +118,7 @@ const insertSuggestion = (state, position, token, completion) => {
}; };
const insertEmoji = (state, position, emojiData) => { const insertEmoji = (state, position, emojiData) => {
const emoji = emojiData.shortname; const emoji = String.fromCodePoint(parseInt(emojiData.unicode, 16));
return state.withMutations(map => { return state.withMutations(map => {
map.update('text', oldText => `${oldText.slice(0, position)}${emoji} ${oldText.slice(position)}`); map.update('text', oldText => `${oldText.slice(0, position)}${emoji} ${oldText.slice(position)}`);

View file

@ -2708,6 +2708,7 @@ button.icon-button.active i.fa-retweet {
margin-left: 2px; margin-left: 2px;
width: 24px; width: 24px;
outline: 0; outline: 0;
cursor: pointer;
&:active, &:active,
&:focus { &:focus {

View file

@ -47,6 +47,7 @@ class Account < ApplicationRecord
include AccountInteractions include AccountInteractions
include Attachmentable include Attachmentable
include Remotable include Remotable
include EmojiHelper
# Local users # Local users
has_one :user, inverse_of: :account has_one :user, inverse_of: :account
@ -240,9 +241,18 @@ class Account < ApplicationRecord
before_create :generate_keys before_create :generate_keys
before_validation :normalize_domain before_validation :normalize_domain
before_validation :prepare_contents, if: :local?
private private
def prepare_contents
display_name&.strip!
note&.strip!
self.display_name = emojify(display_name)
self.note = emojify(note)
end
def generate_keys def generate_keys
return unless local? return unless local?

View file

@ -29,6 +29,7 @@ class Status < ApplicationRecord
include Streamable include Streamable
include Cacheable include Cacheable
include StatusThreadingConcern include StatusThreadingConcern
include EmojiHelper
enum visibility: [:public, :unlisted, :private, :direct], _suffix: :visibility enum visibility: [:public, :unlisted, :private, :direct], _suffix: :visibility
@ -120,7 +121,7 @@ class Status < ApplicationRecord
!sensitive? && media_attachments.any? !sensitive? && media_attachments.any?
end end
before_validation :prepare_contents before_validation :prepare_contents, if: :local?
before_validation :set_reblog before_validation :set_reblog
before_validation :set_visibility before_validation :set_visibility
before_validation :set_conversation before_validation :set_conversation
@ -241,6 +242,9 @@ class Status < ApplicationRecord
def prepare_contents def prepare_contents
text&.strip! text&.strip!
spoiler_text&.strip! spoiler_text&.strip!
self.text = emojify(text)
self.spoiler_text = emojify(spoiler_text)
end end
def set_reblog def set_reblog

View file

@ -21,6 +21,7 @@ class PostStatusService < BaseService
media = validate_media!(options[:media_ids]) media = validate_media!(options[:media_ids])
status = nil status = nil
ApplicationRecord.transaction do ApplicationRecord.transaction do
status = account.statuses.create!(text: text, status = account.statuses.create!(text: text,
thread: in_reply_to, thread: in_reply_to,
@ -31,6 +32,7 @@ class PostStatusService < BaseService
application: options[:application]) application: options[:application])
attach_media(status, media) attach_media(status, media)
end end
process_mentions_service.call(status) process_mentions_service.call(status)
process_hashtags_service.call(status) process_hashtags_service.call(status)

View file

@ -0,0 +1,15 @@
require 'rails_helper'
RSpec.describe EmojiHelper, type: :helper do
describe '#emojify' do
it 'converts shortcodes to unicode' do
text = ':book: Book'
expect(emojify(text)).to eq '📖 Book'
end
it 'does not convert shortcodes that are part of a string into unicode' do
text = ':see_no_evil::hear_no_evil::speak_no_evil:'
expect(emojify(text)).to eq text
end
end
end

View file

@ -1,5 +0,0 @@
require 'rails_helper'
RSpec.describe RoutingHelper, type: :helper do
end

View file

@ -2,32 +2,6 @@ import { expect } from 'chai';
import emojify from '../../../app/javascript/mastodon/emoji'; import emojify from '../../../app/javascript/mastodon/emoji';
describe('emojify', () => { describe('emojify', () => {
it('does a basic emojify', () => {
expect(emojify(':smile:')).to.equal(
'<img draggable="false" class="emojione" alt="😄" title=":smile:" src="/emoji/1f604.svg" />');
});
it('does a double emojify', () => {
expect(emojify(':smile: and :wink:')).to.equal(
'<img draggable="false" class="emojione" alt="😄" title=":smile:" src="/emoji/1f604.svg" /> and <img draggable="false" class="emojione" alt="😉" title=":wink:" src="/emoji/1f609.svg" />');
});
it('works with random colons', () => {
expect(emojify(':smile: : :wink:')).to.equal(
'<img draggable="false" class="emojione" alt="😄" title=":smile:" src="/emoji/1f604.svg" /> : <img draggable="false" class="emojione" alt="😉" title=":wink:" src="/emoji/1f609.svg" />');
expect(emojify(':smile::::wink:')).to.equal(
'<img draggable="false" class="emojione" alt="😄" title=":smile:" src="/emoji/1f604.svg" />::<img draggable="false" class="emojione" alt="😉" title=":wink:" src="/emoji/1f609.svg" />');
expect(emojify(':smile:::::wink:')).to.equal(
'<img draggable="false" class="emojione" alt="😄" title=":smile:" src="/emoji/1f604.svg" />:::<img draggable="false" class="emojione" alt="😉" title=":wink:" src="/emoji/1f609.svg" />');
});
it('works with tags', () => {
expect(emojify('<p>:smile:</p>')).to.equal(
'<p><img draggable="false" class="emojione" alt="😄" title=":smile:" src="/emoji/1f604.svg" /></p>');
expect(emojify('<p>:smile:</p> and <p>:wink:</p>')).to.equal(
'<p><img draggable="false" class="emojione" alt="😄" title=":smile:" src="/emoji/1f604.svg" /></p> and <p><img draggable="false" class="emojione" alt="😉" title=":wink:" src="/emoji/1f609.svg" /></p>');
});
it('ignores unknown shortcodes', () => { it('ignores unknown shortcodes', () => {
expect(emojify(':foobarbazfake:')).to.equal(':foobarbazfake:'); expect(emojify(':foobarbazfake:')).to.equal(':foobarbazfake:');
}); });
@ -46,11 +20,6 @@ describe('emojify', () => {
expect(emojify(':smile')).to.equal(':smile'); expect(emojify(':smile')).to.equal(':smile');
}); });
it('does two emoji next to each other', () => {
expect(emojify(':smile::wink:')).to.equal(
'<img draggable="false" class="emojione" alt="😄" title=":smile:" src="/emoji/1f604.svg" /><img draggable="false" class="emojione" alt="😉" title=":wink:" src="/emoji/1f609.svg" />');
});
it('does unicode', () => { it('does unicode', () => {
expect(emojify('\uD83D\uDC69\u200D\uD83D\uDC69\u200D\uD83D\uDC66\u200D\uD83D\uDC66')).to.equal( expect(emojify('\uD83D\uDC69\u200D\uD83D\uDC69\u200D\uD83D\uDC66\u200D\uD83D\uDC66')).to.equal(
'<img draggable="false" class="emojione" alt="👩‍👩‍👦‍👦" title=":family_wwbb:" src="/emoji/1f469-1f469-1f466-1f466.svg" />'); '<img draggable="false" class="emojione" alt="👩‍👩‍👦‍👦" title=":family_wwbb:" src="/emoji/1f469-1f469-1f466-1f466.svg" />');
@ -72,12 +41,7 @@ describe('emojify', () => {
'foo <img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg" /> <img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/0023-20e3.svg" /> bar'); 'foo <img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg" /> <img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/0023-20e3.svg" /> bar');
}); });
it('does mixed unicode and shortnames', () => {
expect(emojify(':smile:#\uFE0F\u20E3:wink:\u2757')).to.equal('<img draggable="false" class="emojione" alt="😄" title=":smile:" src="/emoji/1f604.svg" /><img draggable="false" class="emojione" alt="#️⃣" title=":hash:" src="/emoji/0023-20e3.svg" /><img draggable="false" class="emojione" alt="😉" title=":wink:" src="/emoji/1f609.svg" /><img draggable="false" class="emojione" alt="❗" title=":exclamation:" src="/emoji/2757.svg" />');
});
it('ignores unicode inside of tags', () => { it('ignores unicode inside of tags', () => {
expect(emojify('<p data-foo="\uD83D\uDC69\uD83D\uDC69\uD83D\uDC66"></p>')).to.equal('<p data-foo="\uD83D\uDC69\uD83D\uDC69\uD83D\uDC66"></p>'); expect(emojify('<p data-foo="\uD83D\uDC69\uD83D\uDC69\uD83D\uDC66"></p>')).to.equal('<p data-foo="\uD83D\uDC69\uD83D\uDC69\uD83D\uDC66"></p>');
}); });
}); });