Improve RTL detection (#3682)

- Use plaintext
- Strip out URLs
- Strip out mentions
- Strip out hashtags
- Strip out whitespace from "overall" count
- Consistent between JS and Ruby
This commit is contained in:
Eugen Rochko 2017-06-10 15:06:50 +02:00 committed by GitHub
parent 4919b89ab8
commit 8015fd7600
5 changed files with 26 additions and 5 deletions

View file

@ -47,11 +47,16 @@ module StreamEntriesHelper
end end
end end
def rtl_status?(status)
status.local? ? rtl?(status.text) : rtl?(strip_tags(status.text))
end
def rtl?(text) def rtl?(text)
text = simplified_text(text)
rtl_characters = /[\p{Hebrew}|\p{Arabic}|\p{Syriac}|\p{Thaana}|\p{Nko}]+/m.match(text) rtl_characters = /[\p{Hebrew}|\p{Arabic}|\p{Syriac}|\p{Thaana}|\p{Nko}]+/m.match(text)
if rtl_characters.present? if rtl_characters.present?
total_size = text.strip.size.to_f total_size = text.size.to_f
rtl_size(rtl_characters.to_a) / total_size > 0.3 rtl_size(rtl_characters.to_a) / total_size > 0.3
else else
false false
@ -60,6 +65,18 @@ module StreamEntriesHelper
private private
def simplified_text(text)
text.dup.tap do |new_text|
URI.extract(new_text).each do |url|
new_text.gsub!(url, '')
end
new_text.gsub!(Account::MENTION_RE, '')
new_text.gsub!(Tag::HASHTAG_RE, '')
new_text.gsub!(/\s+/, '')
end
end
def rtl_size(characters) def rtl_size(characters)
characters.reduce(0) { |acc, elem| acc + elem.size }.to_f characters.reduce(0) { |acc, elem| acc + elem.size }.to_f
end end

View file

@ -100,7 +100,7 @@ class StatusContent extends React.PureComponent {
const spoilerContent = { __html: emojify(escapeTextContentForBrowser(status.get('spoiler_text', ''))) }; const spoilerContent = { __html: emojify(escapeTextContentForBrowser(status.get('spoiler_text', ''))) };
const directionStyle = { direction: 'ltr' }; const directionStyle = { direction: 'ltr' };
if (isRtl(status.get('content'))) { if (isRtl(status.get('search_index'))) {
directionStyle.direction = 'rtl'; directionStyle.direction = 'rtl';
} }

View file

@ -17,11 +17,15 @@ export function isRtl(text) {
return false; return false;
} }
text = text.replace(/(?:^|[^\/\w])@([a-z0-9_]+(@[a-z0-9\.\-]+)?)/ig, '');
text = text.replace(/(?:^|[^\/\w])#([\S]+)/ig, '');
text = text.replace(/\s+/g, '');
const matches = text.match(rtlChars); const matches = text.match(rtlChars);
if (!matches) { if (!matches) {
return false; return false;
} }
return matches.length / text.trim().length > 0.3; return matches.length / text.length > 0.3;
}; };

View file

@ -12,7 +12,7 @@
%p{ style: 'margin-bottom: 0' }< %p{ style: 'margin-bottom: 0' }<
%span.p-summary> #{status.spoiler_text}&nbsp; %span.p-summary> #{status.spoiler_text}&nbsp;
%a.status__content__spoiler-link{ href: '#' }= t('statuses.show_more') %a.status__content__spoiler-link{ href: '#' }= t('statuses.show_more')
.e-content{ lang: status.language, style: "display: #{status.spoiler_text? ? 'none' : 'block'}; direction: #{rtl?(status.content) ? 'rtl' : 'ltr'}" }= Formatter.instance.format(status) .e-content{ lang: status.language, style: "display: #{status.spoiler_text? ? 'none' : 'block'}; direction: #{rtl_status?(status) ? 'rtl' : 'ltr'}" }= Formatter.instance.format(status)
- unless status.media_attachments.empty? - unless status.media_attachments.empty?
- if status.media_attachments.first.video? - if status.media_attachments.first.video?

View file

@ -18,7 +18,7 @@
%p{ style: 'margin-bottom: 0' }< %p{ style: 'margin-bottom: 0' }<
%span.p-summary> #{status.spoiler_text}&nbsp; %span.p-summary> #{status.spoiler_text}&nbsp;
%a.status__content__spoiler-link{ href: '#' }= t('statuses.show_more') %a.status__content__spoiler-link{ href: '#' }= t('statuses.show_more')
.e-content{ lang: status.language, style: "display: #{status.spoiler_text? ? 'none' : 'block'}; direction: #{rtl?(status.content) ? 'rtl' : 'ltr'}" }= Formatter.instance.format(status) .e-content{ lang: status.language, style: "display: #{status.spoiler_text? ? 'none' : 'block'}; direction: #{rtl_status?(status) ? 'rtl' : 'ltr'}" }= Formatter.instance.format(status)
- unless status.media_attachments.empty? - unless status.media_attachments.empty?
.status__attachments .status__attachments