Merge branch 'master' of github.com:tootsuite/mastodon

.babelrc

@@ -4,7 +4,6 @@
     [
       "env",
       {
-        "debug": true,
         "exclude": ["transform-async-to-generator", "transform-regenerator"],
         "loose": true,
         "modules": false,

.circleci/config.yml

@@ -0,0 +1,193 @@
+version: 2
+
+aliases:
+  - &defaults
+    docker:
+      - image: circleci/ruby:2.5.1-stretch-node
+        environment: &ruby_environment
+          BUNDLE_APP_CONFIG: ./.bundle/
+          DB_HOST: localhost
+          DB_USER: root
+          RAILS_ENV: test
+          PARALLEL_TEST_PROCESSORS: 4
+          ALLOW_NOPAM: true
+          CONTINUOUS_INTEGRATION: true
+          DISABLE_SIMPLECOV: true
+    working_directory: ~/projects/mastodon/
+
+  - &attach_workspace
+    attach_workspace:
+      at: ~/projects/
+
+  - &persist_to_workspace
+    persist_to_workspace:
+      root: ~/projects/
+      paths:
+        - ./mastodon/
+
+  - &restore_ruby_dependencies
+    restore_cache:
+      keys:
+        - v2-ruby-dependencies-{{ checksum "/tmp/.ruby-version" }}-{{ checksum "Gemfile.lock" }}
+        - v2-ruby-dependencies-{{ checksum "/tmp/.ruby-version" }}-
+        - v2-ruby-dependencies-
+
+  - &install_steps
+    steps:
+      - checkout
+      - *attach_workspace
+
+      - restore_cache:
+          keys:
+            - v1-node-dependencies-{{ checksum "yarn.lock" }}
+            - v1-node-dependencies-
+      - run: yarn install --frozen-lockfile
+      - save_cache:
+          key: v1-node-dependencies-{{ checksum "yarn.lock" }}
+          paths:
+            - ./node_modules/
+
+      - *persist_to_workspace
+
+  - &install_system_dependencies
+      run:
+        name: Install system dependencies
+        command: |
+          sudo apt-get update
+          sudo apt-get install -y libicu-dev libidn11-dev libprotobuf-dev protobuf-compiler
+
+  - &install_ruby_dependencies
+      steps:
+        - *attach_workspace
+
+        - *install_system_dependencies
+
+        - run: ruby -e 'puts RUBY_VERSION' | tee /tmp/.ruby-version
+        - *restore_ruby_dependencies
+        - run: bundle install --clean --jobs 16 --path ./vendor/bundle/ --retry 3 --with pam_authentication --without development production
+        - save_cache:
+            key: v2-ruby-dependencies-{{ checksum "/tmp/.ruby-version" }}-{{ checksum "Gemfile.lock" }}
+            paths:
+              - ./.bundle/
+              - ./vendor/bundle/
+
+  - &test_steps
+      steps:
+        - *attach_workspace
+
+        - *install_system_dependencies
+        - run: sudo apt-get install -y ffmpeg
+
+        - run: ruby -e 'puts RUBY_VERSION' | tee /tmp/.ruby-version
+        - *restore_ruby_dependencies
+
+        - restore_cache:
+            keys:
+              - precompiled-assets-{{ .Branch }}-{{ .Revision }}
+              - precompiled-assets-{{ .Branch }}-
+              - precompiled-assets-
+
+        - run:
+            name: Prepare Tests
+            command: ./bin/rails parallel:create parallel:load_schema parallel:prepare
+        - run:
+            name: Run Tests
+            command: ./bin/retry bundle exec parallel_test ./spec/ --group-by filesize --type rspec
+
+jobs:
+  install:
+    <<: *defaults
+    <<: *install_steps
+
+  install-ruby2.5:
+    <<: *defaults
+    <<: *install_ruby_dependencies
+
+  install-ruby2.4:
+    <<: *defaults
+    docker:
+      - image: circleci/ruby:2.4.4-stretch-node
+        environment: *ruby_environment
+    <<: *install_ruby_dependencies
+
+  build:
+    <<: *defaults
+    steps:
+      - *attach_workspace
+      - *install_system_dependencies
+      - run: ruby -e 'puts RUBY_VERSION' | tee /tmp/.ruby-version
+      - *restore_ruby_dependencies
+      - run: ./bin/rails assets:precompile
+      - save_cache:
+          key: precompiled-assets-{{ .Branch }}-{{ .Revision }}
+          paths:
+            - ./public/assets
+            - ./public/packs-test/
+
+  test-ruby2.5:
+    <<: *defaults
+    docker:
+      - image: circleci/ruby:2.5.1-stretch-node
+        environment: *ruby_environment
+      - image: circleci/postgres:10.3-alpine
+        environment:
+          POSTGRES_USER: root
+      - image: circleci/redis:4.0.9-alpine
+    <<: *test_steps
+
+  test-ruby2.4:
+    <<: *defaults
+    docker:
+      - image: circleci/ruby:2.4.4-stretch-node
+        environment: *ruby_environment
+      - image: circleci/postgres:10.3-alpine
+        environment:
+          POSTGRES_USER: root
+      - image: circleci/redis:4.0.9-alpine
+    <<: *test_steps
+
+  test-webui:
+    <<: *defaults
+    docker:
+      - image: circleci/node:8.11.1-stretch
+    steps:
+      - *attach_workspace
+      - run: ./bin/retry yarn test:jest
+
+  check-i18n:
+    <<: *defaults
+    steps:
+      - *attach_workspace
+      - run: ruby -e 'puts RUBY_VERSION' | tee /tmp/.ruby-version
+      - *restore_ruby_dependencies
+      - run: bundle exec i18n-tasks check-normalized
+      - run: bundle exec i18n-tasks unused
+
+workflows:
+  version: 2
+  build-and-test:
+    jobs:
+      - install
+      - install-ruby2.5:
+          requires:
+            - install
+      - install-ruby2.4:
+          requires:
+            - install
+      - build:
+          requires:
+            - install-ruby2.5
+      - test-ruby2.5:
+          requires:
+            - install-ruby2.5
+            - build
+      - test-ruby2.4:
+          requires:
+            - install-ruby2.4
+            - build
+      - test-webui:
+          requires:
+            - install
+      - check-i18n:
+          requires:
+            - install-ruby2.5

.dockerignore

@@ -11,3 +11,4 @@ vendor/bundle
 *~
 postgres
 redis
+elasticsearch

.env.nanobox

@@ -14,9 +14,9 @@ DB_PORT=5432
 DATABASE_URL=postgresql://$DATA_DB_USER:$DATA_DB_PASS@$DATA_DB_HOST/gonano
 
 # Optional ElasticSearch configuration
-# ES_ENABLED=true
-# ES_HOST=localhost
-# ES_PORT=9200
+ES_ENABLED=true
+ES_HOST=$DATA_ELASTIC_HOST
+ES_PORT=9200
 
 # Optimizations
 LD_PRELOAD=/data/lib/libjemalloc.so

.env.production.sample

@@ -81,6 +81,10 @@ SMTP_FROM_ADDRESS=notifications@example.com
 # PAPERCLIP_ROOT_URL=/system
 
 # Optional asset host for multi-server setups
+# The asset host must allow cross origin request from WEB_DOMAIN or LOCAL_DOMAIN
+# if WEB_DOMAIN is not set. For example, the server may have the
+# following header field:
+# Access-Control-Allow-Origin: https://example.com/
 # CDN_HOST=https://assets.example.com
 
 # S3 (optional)
@@ -109,6 +113,8 @@ SMTP_FROM_ADDRESS=notifications@example.com
 # For Keystone V3, the value for SWIFT_TENANT should be the project name
 # SWIFT_TENANT=
 # SWIFT_PASSWORD=
+# Some OpenStack V3 providers require PROJECT_ID (optional)
+# SWIFT_PROJECT_ID=
 # Keystone V2 and V3 URLs are supported. Use a V3 URL if possible to avoid
 # issues with token rate-limiting during high load.
 # SWIFT_AUTH_URL=
@@ -155,8 +161,8 @@ STREAMING_CLUSTER_NUM=1
 # The pam environment variable "email" is provided by:
 # https://github.com/devkral/pam_email_extractor
 # PAM_ENABLED=true
-# Fallback Suffix for email address generation (nil by default)
-# PAM_DEFAULT_SUFFIX=pam
+# Fallback email domain for email address generation (LOCAL_DOMAIN by default)
+# PAM_EMAIL_DOMAIN=example.com
 # Name of the pam service (pam "auth" section is evaluated)
 # PAM_DEFAULT_SERVICE=rpam
 # Name of the pam service used for checking if an user can register (pam "account" section is evaluated) (nil (disabled) by default)
@@ -210,3 +216,10 @@ STREAMING_CLUSTER_NUM=1
 # SAML_UID_ATTRIBUTE="urn:oid:0.9.2342.19200300.100.1.1"
 # SAML_ATTRIBUTES_STATEMENTS_VERIFIED=
 # SAML_ATTRIBUTES_STATEMENTS_VERIFIED_EMAIL=
+
+# Use HTTP proxy for outgoing request (optional)
+# http_proxy=http://gateway.local:8118
+# Access control for hidden service.
+# ALLOW_ACCESS_TO_HIDDEN_SERVICE=true
+# If you use transparent proxy to access to hidden service, uncomment following for skipping private address check.
+# HIDDEN_SERVICE_VIA_TRANSPARENT_PROXY=true

.env.test

@@ -1,3 +1,9 @@
+# Node.js
+NODE_ENV=test
 # Federation
 LOCAL_DOMAIN=cb6e6126.ngrok.io
 LOCAL_HTTPS=true
+# test pam authentication
+PAM_ENABLED=true
+PAM_DEFAULT_SERVICE=pam_test
+PAM_CONTROLLED_SERVICE=pam_test_controlled

.eslintrc.yml

@@ -13,6 +13,7 @@ plugins:
 - react
 - jsx-a11y
 - import
+- promise
 
 parserOptions:
   sourceType: module
@@ -152,3 +153,5 @@ rules:
     - "app/javascript/**/__tests__/**"
   import/no-unresolved: error
   import/no-webpack-loader-syntax: error
+
+  promise/catch-or-return: error

.github/ISSUE_TEMPLATE.md → .github/ISSUE_TEMPLATE/bug_report.md

@@ -1,3 +1,9 @@
+---
+name: Bug Report
+about: Create a report to help us improve
+
+---
+
 [Issue text goes here].
 
 * * * *

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,11 @@
+---
+name: Feature Request
+about: Suggest an idea for this project
+
+---
+
+[Issue text goes here].
+
+* * * *
+
+- [ ] I searched or browsed the repo’s other issues to ensure this is not a duplicate.

.gitignore

@@ -36,9 +36,10 @@ config/deploy/*
 .vscode/
 .idea/
 
-# Ignore postgres + redis volume optionally created by docker-compose
+# Ignore postgres + redis + elasticsearch volume optionally created by docker-compose
 postgres
 redis
+elasticsearch
 
 # Ignore Apple files
 .DS_Store

.ruby-version

@@ -1 +1 @@
-2.5.0
+2.5.1

.travis.yml

@@ -1,59 +0,0 @@
-language: ruby
-cache:
-  bundler: true
-  yarn: true
-  directories:
-    - node_modules
-    - public/assets
-    - public/packs-test
-    - tmp/cache/babel-loader
-dist: trusty
-sudo: false
-branches:
-  only:
-    - master
-
-notifications:
-  email: false
-
-env:
-  global:
-    - LOCAL_DOMAIN=cb6e6126.ngrok.io
-    - LOCAL_HTTPS=true
-    - RAILS_ENV=test
-    - NOKOGIRI_USE_SYSTEM_LIBRARIES=true
-    - PARALLEL_TEST_PROCESSORS=2
-
-addons:
-  postgresql: 9.4
-  apt:
-    sources:
-      - trusty-media
-      - sourceline: deb https://dl.yarnpkg.com/debian/ stable main
-        key_url: https://dl.yarnpkg.com/debian/pubkey.gpg
-    packages:
-      - ffmpeg
-      - libicu-dev
-      - libprotobuf-dev
-      - protobuf-compiler
-      - yarn
-
-rvm:
-  - 2.4.2
-  - 2.5.0
-
-services:
-  - redis-server
-
-install:
-  - nvm install
-  - bundle install --path=vendor/bundle --without development production --retry=3 --jobs=16
-  - yarn install
-
-before_script:
-  - ./bin/rails parallel:create parallel:load_schema parallel:prepare assets:precompile
-
-script:
-  - travis_retry bundle exec parallel_test spec/ --group-by filesize --type rspec
-  - yarn run test:jest
-  - bundle exec i18n-tasks check-normalized && bundle exec i18n-tasks unused

CONTRIBUTING.md

@@ -49,3 +49,8 @@ It is expected that you have a working development environment set up (see back-
 * If you are introducing new strings, they must be using localization methods
 
 If the JavaScript or CSS assets won't compile due to a syntax error, it's a good sign that the pull request isn't ready for submission yet.
+
+## Translate
+
+You can contribute to translating Mastodon via Weblate at [weblate.joinmastodon.org](https://weblate.joinmastodon.org/).
+[![Mastodon translation statistics by language](https://weblate.joinmastodon.org/widgets/mastodon/-/multi-auto.svg)](https://weblate.joinmastodon.org/)

Dockerfile

@@ -1,4 +1,4 @@
-FROM ruby:2.4.3-alpine3.6
+FROM ruby:2.4.4-alpine3.6
 
 LABEL maintainer="https://github.com/tootsuite/mastodon" \
       description="Your self-hosted, globally interconnected microblogging community"

Gemfile

@@ -3,95 +3,101 @@
 source 'https://rubygems.org'
 ruby '>= 2.3.0', '< 2.6.0'
 
-gem 'pkg-config', '~> 1.2'
+gem 'pkg-config', '~> 1.3'
 
-gem 'puma', '~> 3.10'
-gem 'rails', '~> 5.1.4'
+gem 'puma', '~> 3.11'
+gem 'rails', '~> 5.2.0'
 
 gem 'hamlit-rails', '~> 0.2'
-gem 'pg', '~> 0.20'
-gem 'pghero', '~> 1.7'
-gem 'dotenv-rails', '~> 2.2'
+gem 'pg', '~> 1.0'
+gem 'pghero', '~> 2.1'
+gem 'dotenv-rails', '~> 2.2', '< 2.3'
 
-gem 'aws-sdk', '~> 2.10', require: false
+gem 'aws-sdk-s3', '~> 1.9', require: false
 gem 'fog-core', '~> 1.45'
-gem 'fog-local', '~> 0.4', require: false
+gem 'fog-local', '~> 0.5', require: false
 gem 'fog-openstack', '~> 0.1', require: false
-gem 'paperclip', '~> 5.1'
+gem 'paperclip', '~> 6.0'
 gem 'paperclip-av-transcoder', '~> 0.6'
 gem 'streamio-ffmpeg', '~> 3.0'
 
 gem 'active_model_serializers', '~> 0.10'
 gem 'addressable', '~> 2.5'
-gem 'bootsnap'
+gem 'bootsnap', '~> 1.3'
 gem 'browser'
-gem 'charlock_holmes', '~> 0.7.5'
+gem 'charlock_holmes', '~> 0.7.6'
 gem 'iso-639'
 gem 'chewy', '~> 5.0'
 gem 'cld3', '~> 3.2.0'
 gem 'devise', '~> 4.4'
 gem 'devise-two-factor', '~> 3.0'
 
-gem 'devise_pam_authenticatable2', '~> 8.0', install_if: -> { ENV['PAM_ENABLED'] == 'true' }
+group :pam_authentication, optional: true do
+  gem 'devise_pam_authenticatable2', '~> 9.1'
+end
+
 gem 'net-ldap', '~> 0.10'
 gem 'omniauth-cas', '~> 1.1'
 gem 'omniauth-saml', '~> 1.10'
 gem 'omniauth', '~> 1.2'
 
-gem 'doorkeeper', '~> 4.2'
+gem 'doorkeeper', '~> 4.2', '< 4.3'
 gem 'fast_blank', '~> 1.0'
 gem 'fastimage'
 gem 'goldfinger', '~> 2.1'
 gem 'hiredis', '~> 0.6'
 gem 'redis-namespace', '~> 1.5'
 gem 'htmlentities', '~> 4.3'
-gem 'http', '~> 3.0'
+gem 'http', '~> 3.2'
 gem 'http_accept_language', '~> 2.1'
-gem 'httplog', '~> 0.99'
+gem 'http_parser.rb', '~> 0.6', git: 'https://github.com/tmm1/http_parser.rb', ref: '54b17ba8c7d8d20a16dfc65d1775241833219cf2'
+gem 'httplog', '~> 1.0'
 gem 'idn-ruby', require: 'idn'
 gem 'kaminari', '~> 1.1'
 gem 'link_header', '~> 0.0'
-gem 'mime-types', '~> 3.1'
+gem 'mime-types', '~> 3.1', require: 'mime/types/columnar'
 gem 'nokogiri', '~> 1.8'
 gem 'nsa', '~> 0.2'
-gem 'oj', '~> 3.3'
+gem 'oj', '~> 3.5'
 gem 'ostatus2', '~> 2.0'
-gem 'ox', '~> 2.8'
+gem 'ox', '~> 2.9'
+gem 'posix-spawn', git: 'https://github.com/rtomayko/posix-spawn', ref: '58465d2e213991f8afb13b984854a49fcdcc980c'
 gem 'pundit', '~> 1.1'
 gem 'premailer-rails'
-gem 'rack-attack', '~> 5.0'
-gem 'rack-cors', '~> 0.4', require: 'rack/cors'
+gem 'rack-attack', '~> 5.2'
+gem 'rack-cors', '~> 1.0', require: 'rack/cors'
 gem 'rack-timeout', '~> 0.4'
-gem 'rails-i18n', '~> 5.0'
+gem 'rails-i18n', '~> 5.1'
 gem 'rails-settings-cached', '~> 0.6'
-gem 'redis', '~> 3.3', require: ['redis', 'redis/connection/hiredis']
+gem 'redis', '~> 4.0', require: ['redis', 'redis/connection/hiredis']
 gem 'mario-redis-lock', '~> 1.2', require: 'redis_lock'
 gem 'rqrcode', '~> 0.10'
-gem 'ruby-oembed', '~> 0.12', require: 'oembed'
 gem 'ruby-progressbar', '~> 1.4'
-gem 'sanitize', '~> 4.4'
-gem 'sidekiq', '~> 5.0'
-gem 'sidekiq-scheduler', '~> 2.1'
+gem 'sanitize', '~> 4.6'
+gem 'sidekiq', '~> 5.1'
+gem 'sidekiq-scheduler', '~> 2.2'
 gem 'sidekiq-unique-jobs', '~> 5.0'
 gem 'sidekiq-bulk', '~>0.1.1'
 gem 'simple-navigation', '~> 4.0'
-gem 'simple_form', '~> 3.4'
+gem 'simple_form', '~> 4.0'
 gem 'sprockets-rails', '~> 3.2', require: 'sprockets/railtie'
-gem 'strong_migrations'
-gem 'tty-command'
-gem 'tty-prompt'
+gem 'stoplight', '~> 2.1.3'
+gem 'strong_migrations', '~> 0.2'
+gem 'tty-command', '~> 0.8', require: false
+gem 'tty-prompt', '~> 0.16', require: false
 gem 'twitter-text', '~> 1.14'
-gem 'tzinfo-data', '~> 1.2017'
-gem 'webpacker', '~> 3.0'
+gem 'tzinfo-data', '~> 1.2018'
+gem 'webpacker', '~> 3.4'
 gem 'webpush'
 
-gem 'json-ld-preloaded', '~> 2.2.1'
-gem 'rdf-normalize', '~> 0.3.1'
+gem 'json-ld', '~> 2.2'
+gem 'rdf-normalize', '~> 0.3'
 
 group :development, :test do
-  gem 'fabrication', '~> 2.18'
+  gem 'fabrication', '~> 2.20'
   gem 'fuubar', '~> 2.2'
   gem 'i18n-tasks', '~> 0.9', require: false
+  gem 'pry-byebug', '~> 3.6'
   gem 'pry-rails', '~> 0.3'
   gem 'rspec-rails', '~> 3.7'
 end
@@ -101,15 +107,15 @@ group :production, :test do
 end
 
 group :test do
-  gem 'capybara', '~> 2.15'
+  gem 'capybara', '~> 2.18'
   gem 'climate_control', '~> 0.2'
-  gem 'faker', '~> 1.7'
+  gem 'faker', '~> 1.8'
   gem 'microformats', '~> 4.0'
   gem 'rails-controller-testing', '~> 1.0'
   gem 'rspec-sidekiq', '~> 3.0'
-  gem 'simplecov', '~> 0.14', require: false
-  gem 'webmock', '~> 3.0'
-  gem 'parallel_tests', '~> 2.17'
+  gem 'simplecov', '~> 0.16', require: false
+  gem 'webmock', '~> 3.3'
+  gem 'parallel_tests', '~> 2.21'
 end
 
 group :development do
@@ -117,22 +123,25 @@ group :development do
   gem 'annotate', '~> 2.7'
   gem 'better_errors', '~> 2.4'
   gem 'binding_of_caller', '~> 0.7'
-  gem 'bullet', '~> 5.5'
+  gem 'bullet', '~> 5.7'
   gem 'letter_opener', '~> 1.4'
   gem 'letter_opener_web', '~> 1.3'
   gem 'memory_profiler'
-  gem 'rubocop', require: false
-  gem 'brakeman', '~> 4.0', require: false
+  gem 'rubocop', '~> 0.55', require: false
+  gem 'brakeman', '~> 4.2', require: false
   gem 'bundler-audit', '~> 0.6', require: false
-  gem 'scss_lint', '~> 0.55', require: false
+  gem 'scss_lint', '~> 0.57', require: false
 
   gem 'capistrano', '~> 3.10'
   gem 'capistrano-rails', '~> 1.3'
   gem 'capistrano-rbenv', '~> 2.1'
   gem 'capistrano-yarn', '~> 2.0'
+
+  gem 'derailed_benchmarks'
+  gem 'stackprof'
 end
 
 group :production do
-  gem 'lograge', '~> 0.7'
+  gem 'lograge', '~> 0.10'
   gem 'redis-rails', '~> 5.0'
 end

Gemfile.lock

@@ -1,25 +1,39 @@
+GIT
+  remote: https://github.com/rtomayko/posix-spawn
+  revision: 58465d2e213991f8afb13b984854a49fcdcc980c
+  ref: 58465d2e213991f8afb13b984854a49fcdcc980c
+  specs:
+    posix-spawn (0.3.13)
+
+GIT
+  remote: https://github.com/tmm1/http_parser.rb
+  revision: 54b17ba8c7d8d20a16dfc65d1775241833219cf2
+  ref: 54b17ba8c7d8d20a16dfc65d1775241833219cf2
+  specs:
+    http_parser.rb (0.6.1)
+
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (5.1.4)
-      actionpack (= 5.1.4)
+    actioncable (5.2.0)
+      actionpack (= 5.2.0)
       nio4r (~> 2.0)
-      websocket-driver (~> 0.6.1)
-    actionmailer (5.1.4)
-      actionpack (= 5.1.4)
-      actionview (= 5.1.4)
-      activejob (= 5.1.4)
+      websocket-driver (>= 0.6.1)
+    actionmailer (5.2.0)
+      actionpack (= 5.2.0)
+      actionview (= 5.2.0)
+      activejob (= 5.2.0)
       mail (~> 2.5, >= 2.5.4)
       rails-dom-testing (~> 2.0)
-    actionpack (5.1.4)
-      actionview (= 5.1.4)
-      activesupport (= 5.1.4)
+    actionpack (5.2.0)
+      actionview (= 5.2.0)
+      activesupport (= 5.2.0)
       rack (~> 2.0)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.0.2)
-    actionview (5.1.4)
-      activesupport (= 5.1.4)
+    actionview (5.2.0)
+      activesupport (= 5.2.0)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
@@ -30,60 +44,71 @@ GEM
       case_transform (>= 0.2)
       jsonapi-renderer (>= 0.1.1.beta1, < 0.3)
     active_record_query_trace (1.5.4)
-    activejob (5.1.4)
-      activesupport (= 5.1.4)
+    activejob (5.2.0)
+      activesupport (= 5.2.0)
       globalid (>= 0.3.6)
-    activemodel (5.1.4)
-      activesupport (= 5.1.4)
-    activerecord (5.1.4)
-      activemodel (= 5.1.4)
-      activesupport (= 5.1.4)
-      arel (~> 8.0)
-    activesupport (5.1.4)
+    activemodel (5.2.0)
+      activesupport (= 5.2.0)
+    activerecord (5.2.0)
+      activemodel (= 5.2.0)
+      activesupport (= 5.2.0)
+      arel (>= 9.0)
+    activestorage (5.2.0)
+      actionpack (= 5.2.0)
+      activerecord (= 5.2.0)
+      marcel (~> 0.3.1)
+    activesupport (5.2.0)
       concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (~> 0.7)
+      i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
     addressable (2.5.2)
       public_suffix (>= 2.0.2, < 4.0)
     airbrussh (1.3.0)
       sshkit (>= 1.6.1, != 1.7.0)
-    annotate (2.7.2)
+    annotate (2.7.3)
       activerecord (>= 3.2, < 6.0)
       rake (>= 10.4, < 13.0)
-    arel (8.0.0)
-    ast (2.3.0)
-    attr_encrypted (3.0.3)
+    arel (9.0.0)
+    ast (2.4.0)
+    attr_encrypted (3.1.0)
       encryptor (~> 3.0.0)
     av (0.9.0)
       cocaine (~> 0.5.3)
-    aws-sdk (2.10.100)
-      aws-sdk-resources (= 2.10.100)
-    aws-sdk-core (2.10.100)
+    aws-partitions (1.80.0)
+    aws-sdk-core (3.19.0)
+      aws-partitions (~> 1.0)
       aws-sigv4 (~> 1.0)
       jmespath (~> 1.0)
-    aws-sdk-resources (2.10.100)
-      aws-sdk-core (= 2.10.100)
+    aws-sdk-kms (1.5.0)
+      aws-sdk-core (~> 3)
+      aws-sigv4 (~> 1.0)
+    aws-sdk-s3 (1.9.1)
+      aws-sdk-core (~> 3)
+      aws-sdk-kms (~> 1)
+      aws-sigv4 (~> 1.0)
     aws-sigv4 (1.0.2)
     bcrypt (3.1.11)
+    benchmark-ips (2.7.2)
     better_errors (2.4.0)
       coderay (>= 1.0.0)
       erubi (>= 1.0.0)
       rack (>= 0.9.0)
     binding_of_caller (0.8.0)
       debug_inspector (>= 0.0.1)
-    bootsnap (1.1.5)
+    bootsnap (1.3.0)
       msgpack (~> 1.0)
-    brakeman (4.0.1)
-    browser (2.5.2)
+    brakeman (4.2.1)
+    browser (2.5.3)
     builder (3.2.3)
-    bullet (5.6.1)
+    bullet (5.7.5)
       activesupport (>= 3.0.0)
-      uniform_notifier (~> 1.10.0)
+      uniform_notifier (~> 1.11.0)
     bundler-audit (0.6.0)
       bundler (~> 1.2)
       thor (~> 0.18)
-    capistrano (3.10.0)
+    byebug (10.0.2)
+    capistrano (3.10.2)
       airbrussh (>= 1.0.0)
       i18n
       rake (>= 10.0.0)
@@ -99,21 +124,21 @@ GEM
       sshkit (~> 1.3)
     capistrano-yarn (2.0.2)
       capistrano (~> 3.0)
-    capybara (2.16.1)
+    capybara (2.18.0)
       addressable
       mini_mime (>= 0.1.3)
       nokogiri (>= 1.3.3)
       rack (>= 1.0.0)
       rack-test (>= 0.5.4)
-      xpath (~> 2.0)
+      xpath (>= 2.0, < 4.0)
     case_transform (0.2)
       activesupport
-    charlock_holmes (0.7.5)
+    charlock_holmes (0.7.6)
     chewy (5.0.0)
       activesupport (>= 4.0)
       elasticsearch (>= 2.0.0)
       elasticsearch-dsl
-    chunky_png (1.3.8)
+    chunky_png (1.3.10)
     cld3 (3.2.2)
       ffi (>= 1.1.0, < 1.10.0)
     climate_control (0.2.0)
@@ -125,62 +150,69 @@ GEM
     connection_pool (2.2.1)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
-    crass (1.0.3)
+    crass (1.0.4)
     css_parser (1.6.0)
       addressable
     debug_inspector (0.0.3)
-    devise (4.4.0)
+    derailed_benchmarks (1.3.4)
+      benchmark-ips (~> 2)
+      get_process_mem (~> 0)
+      heapy (~> 0)
+      memory_profiler (~> 0)
+      rack (>= 1)
+      rake (> 10, < 13)
+      thor (~> 0.19)
+    devise (4.4.3)
       bcrypt (~> 3.0)
       orm_adapter (~> 0.1)
-      railties (>= 4.1.0, < 5.2)
+      railties (>= 4.1.0, < 6.0)
       responders
       warden (~> 1.2.3)
-    devise-two-factor (3.0.2)
-      activesupport (< 5.2)
+    devise-two-factor (3.0.3)
+      activesupport (< 5.3)
       attr_encrypted (>= 1.3, < 4, != 2)
       devise (~> 4.0)
-      railties (< 5.2)
+      railties (< 5.3)
       rotp (~> 2.0)
-    devise_pam_authenticatable2 (8.0.1)
+    devise_pam_authenticatable2 (9.1.0)
       devise (>= 4.0.0)
-      rpam2 (~> 3.0)
+      rpam2 (~> 4.0)
     diff-lcs (1.3)
-    docile (1.1.5)
-    domain_name (0.5.20170404)
+    docile (1.3.0)
+    domain_name (0.5.20180417)
       unf (>= 0.0.5, < 1.0.0)
     doorkeeper (4.2.6)
       railties (>= 4.2)
-    dotenv (2.2.1)
-    dotenv-rails (2.2.1)
-      dotenv (= 2.2.1)
-      railties (>= 3.2, < 5.2)
-    easy_translate (0.5.0)
-      json
+    dotenv (2.2.2)
+    dotenv-rails (2.2.2)
+      dotenv (= 2.2.2)
+      railties (>= 3.2, < 6.0)
+    easy_translate (0.5.1)
       thread
       thread_safe
-    elasticsearch (6.0.1)
-      elasticsearch-api (= 6.0.1)
-      elasticsearch-transport (= 6.0.1)
-    elasticsearch-api (6.0.1)
+    elasticsearch (6.0.2)
+      elasticsearch-api (= 6.0.2)
+      elasticsearch-transport (= 6.0.2)
+    elasticsearch-api (6.0.2)
       multi_json
     elasticsearch-dsl (0.1.5)
-    elasticsearch-transport (6.0.1)
+    elasticsearch-transport (6.0.2)
       faraday
       multi_json
     encryptor (3.0.0)
     equatable (0.5.0)
-    erubi (1.7.0)
-    et-orbi (1.0.8)
+    erubi (1.7.1)
+    et-orbi (1.1.0)
       tzinfo
-    excon (0.59.0)
-    fabrication (2.18.0)
-    faker (1.8.4)
-      i18n (~> 0.5)
-    faraday (0.14.0)
+    excon (0.62.0)
+    fabrication (2.20.1)
+    faker (1.8.7)
+      i18n (>= 0.7)
+    faraday (0.15.0)
       multipart-post (>= 1.2, < 3)
     fast_blank (1.0.0)
     fastimage (2.1.1)
-    ffi (1.9.18)
+    ffi (1.9.23)
     fog-core (1.45.0)
       builder
       excon (~> 0.58)
@@ -188,16 +220,17 @@ GEM
     fog-json (1.0.2)
       fog-core (~> 1.0)
       multi_json (~> 1.10)
-    fog-local (0.4.0)
-      fog-core (~> 1.27)
-    fog-openstack (0.1.22)
-      fog-core (>= 1.40)
+    fog-local (0.5.0)
+      fog-core (>= 1.27, < 3.0)
+    fog-openstack (0.1.25)
+      fog-core (~> 1.40)
       fog-json (>= 1.0)
       ipaddress (>= 0.8)
     formatador (0.2.5)
-    fuubar (2.2.0)
+    fuubar (2.3.1)
       rspec-core (~> 3.0)
       ruby-progressbar (~> 1.4)
+    get_process_mem (0.2.1)
     globalid (0.4.1)
       activesupport (>= 4.2.0)
     goldfinger (2.1.0)
@@ -205,7 +238,7 @@ GEM
       http (~> 3.0)
       nokogiri (~> 1.8)
       oj (~> 3.0)
-    hamlit (2.8.5)
+    hamlit (2.8.8)
       temple (>= 0.8.0)
       thor
       tilt
@@ -218,48 +251,44 @@ GEM
       concurrent-ruby (~> 1.0)
     hashdiff (0.3.7)
     hashie (3.5.7)
+    heapy (0.1.3)
     highline (1.7.10)
     hiredis (0.6.1)
     hitimes (1.2.6)
     hkdf (0.3.0)
     htmlentities (4.3.4)
-    http (3.0.0)
+    http (3.2.0)
       addressable (~> 2.3)
       http-cookie (~> 1.0)
-      http-form_data (>= 2.0.0.pre.pre2, < 3)
+      http-form_data (~> 2.0)
       http_parser.rb (~> 0.6.0)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
-    http-form_data (2.0.0)
+    http-form_data (2.1.0)
     http_accept_language (2.1.1)
-    http_parser.rb (0.6.0)
-    httplog (0.99.7)
-      colorize
-      rack
-    i18n (0.9.3)
+    httplog (1.0.2)
+      colorize (~> 0.8)
+      rack (>= 1.0)
+    i18n (1.0.1)
       concurrent-ruby (~> 1.0)
-    i18n-tasks (0.9.19)
+    i18n-tasks (0.9.21)
       activesupport (>= 4.0.2)
       ast (>= 2.1.0)
-      easy_translate (>= 0.5.0)
+      easy_translate (>= 0.5.1)
       erubi
       highline (>= 1.7.3)
       i18n
       parser (>= 2.2.3.0)
-      rainbow (~> 2.2)
+      rainbow (>= 2.2.2, < 4.0)
       terminal-table (>= 1.5.1)
     idn-ruby (0.1.0)
     ipaddress (0.8.3)
     iso-639 (0.2.8)
-    jmespath (1.3.1)
+    jmespath (1.4.0)
     json (2.1.0)
-    json-ld (2.1.7)
+    json-ld (2.2.1)
       multi_json (~> 1.12)
-      rdf (~> 2.2, >= 2.2.8)
-    json-ld-preloaded (2.2.2)
-      json-ld (~> 2.1, >= 2.1.5)
-      multi_json (~> 1.11)
-      rdf (~> 2.2)
+      rdf (>= 2.2.8, < 4.0)
     jsonapi-renderer (0.2.0)
     jwt (2.1.0)
     kaminari (1.1.1)
@@ -276,25 +305,27 @@ GEM
     kaminari-core (1.1.1)
     launchy (2.4.3)
       addressable (~> 2.3)
-    letter_opener (1.4.1)
+    letter_opener (1.6.0)
       launchy (~> 2.2)
-    letter_opener_web (1.3.1)
+    letter_opener_web (1.3.4)
       actionmailer (>= 3.2)
       letter_opener (~> 1.0)
       railties (>= 3.2)
     link_header (0.0.8)
-    lograge (0.7.1)
-      actionpack (>= 4, < 5.2)
-      activesupport (>= 4, < 5.2)
-      railties (>= 4, < 5.2)
+    lograge (0.10.0)
+      actionpack (>= 4)
+      activesupport (>= 4)
+      railties (>= 4)
       request_store (~> 1.0)
-    loofah (2.1.1)
+    loofah (2.2.2)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     mail (2.7.0)
       mini_mime (>= 0.1.1)
-    mario-redis-lock (1.2.0)
-      redis (~> 3, >= 3.0.5)
+    marcel (0.3.2)
+      mimemagic (~> 0.3.2)
+    mario-redis-lock (1.2.1)
+      redis (>= 3.0.5)
     memory_profiler (0.9.10)
     method_source (0.9.0)
     microformats (4.0.7)
@@ -307,25 +338,25 @@ GEM
     mini_mime (1.0.0)
     mini_portile2 (2.3.0)
     minitest (5.11.3)
-    msgpack (1.1.0)
-    multi_json (1.12.2)
+    msgpack (1.2.4)
+    multi_json (1.13.1)
     multipart-post (2.0.0)
     necromancer (0.4.0)
     net-ldap (0.16.1)
     net-scp (1.2.1)
       net-ssh (>= 2.6.5)
     net-ssh (4.2.0)
-    nio4r (2.1.0)
-    nokogiri (1.8.1)
+    nio4r (2.3.0)
+    nokogiri (1.8.2)
       mini_portile2 (~> 2.3.0)
-    nokogumbo (1.4.13)
+    nokogumbo (1.5.0)
       nokogiri
     nsa (0.2.4)
       activesupport (>= 4.2, < 6)
       concurrent-ruby (~> 1.0.0)
       sidekiq (>= 3.5.0)
       statsd-ruby (~> 1.2.0)
-    oj (3.3.10)
+    oj (3.5.1)
     omniauth (1.8.1)
       hashie (>= 3.4.6, < 3.6.0)
       rack (>= 1.6.2, < 3)
@@ -341,68 +372,72 @@ GEM
       addressable (~> 2.5)
       http (~> 3.0)
       nokogiri (~> 1.8)
-    ox (2.8.2)
-    paperclip (5.2.1)
+    ox (2.9.2)
+    paperclip (6.0.0)
       activemodel (>= 4.2.0)
       activesupport (>= 4.2.0)
-      cocaine (~> 0.5.5)
       mime-types
       mimemagic (~> 0.3.0)
+      terrapin (~> 0.6.0)
     paperclip-av-transcoder (0.6.4)
       av (~> 0.9.0)
       paperclip (>= 2.5.2)
-    parallel (1.12.0)
-    parallel_tests (2.19.0)
+    parallel (1.12.1)
+    parallel_tests (2.21.3)
       parallel
-    parser (2.4.0.2)
-      ast (~> 2.3)
+    parser (2.5.1.0)
+      ast (~> 2.4.0)
     pastel (0.7.2)
       equatable (~> 0.5.0)
       tty-color (~> 0.4.0)
-    pg (0.21.0)
-    pghero (1.7.0)
+    pg (1.0.0)
+    pghero (2.1.0)
       activerecord
-    pkg-config (1.2.8)
+    pkg-config (1.3.0)
     powerpack (0.1.1)
     premailer (1.11.1)
       addressable
       css_parser (>= 1.6.0)
       htmlentities (>= 4.0.0)
-    premailer-rails (1.10.1)
+    premailer-rails (1.10.2)
       actionmailer (>= 3, < 6)
       premailer (~> 1.7, >= 1.7.9)
     private_address_check (0.4.1)
     pry (0.11.3)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
+    pry-byebug (3.6.0)
+      byebug (~> 10.0)
+      pry (~> 0.10)
     pry-rails (0.3.6)
       pry (>= 0.10.4)
-    public_suffix (3.0.1)
-    puma (3.11.0)
+    public_suffix (3.0.2)
+    puma (3.11.4)
     pundit (1.1.0)
       activesupport (>= 3.0.0)
-    rack (2.0.3)
-    rack-attack (5.0.1)
+    rack (2.0.4)
+    rack-attack (5.2.0)
       rack
-    rack-cors (0.4.1)
-    rack-protection (2.0.0)
+    rack-cors (1.0.2)
+    rack-protection (2.0.1)
       rack
-    rack-proxy (0.6.2)
+    rack-proxy (0.6.4)
       rack
-    rack-test (0.8.2)
+    rack-test (1.0.0)
       rack (>= 1.0, < 3)
     rack-timeout (0.4.2)
-    rails (5.1.4)
-      actioncable (= 5.1.4)
-      actionmailer (= 5.1.4)
-      actionpack (= 5.1.4)
-      actionview (= 5.1.4)
-      activejob (= 5.1.4)
-      activemodel (= 5.1.4)
-      activerecord (= 5.1.4)
-      activesupport (= 5.1.4)
+    rails (5.2.0)
+      actioncable (= 5.2.0)
+      actionmailer (= 5.2.0)
+      actionpack (= 5.2.0)
+      actionview (= 5.2.0)
+      activejob (= 5.2.0)
+      activemodel (= 5.2.0)
+      activerecord (= 5.2.0)
+      activestorage (= 5.2.0)
+      activesupport (= 5.2.0)
       bundler (>= 1.3.0)
-      railties (= 5.1.4)
+      railties (= 5.2.0)
       sprockets-rails (>= 2.0.0)
     rails-controller-testing (1.0.2)
       actionpack (~> 5.x, >= 5.0.1)
@@ -411,31 +446,30 @@ GEM
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.0.3)
-      loofah (~> 2.0)
-    rails-i18n (5.0.4)
-      i18n (~> 0.7)
-      railties (~> 5.0)
+    rails-html-sanitizer (1.0.4)
+      loofah (~> 2.2, >= 2.2.2)
+    rails-i18n (5.1.1)
+      i18n (>= 0.7, < 2)
+      railties (>= 5.0, < 6)
     rails-settings-cached (0.6.6)
       rails (>= 4.2.0)
-    railties (5.1.4)
-      actionpack (= 5.1.4)
-      activesupport (= 5.1.4)
+    railties (5.2.0)
+      actionpack (= 5.2.0)
+      activesupport (= 5.2.0)
       method_source
       rake (>= 0.8.7)
       thor (>= 0.18.1, < 2.0)
-    rainbow (2.2.2)
-      rake
-    rake (12.3.0)
-    rb-fsevent (0.10.2)
+    rainbow (3.0.0)
+    rake (12.3.1)
+    rb-fsevent (0.10.3)
     rb-inotify (0.9.10)
       ffi (>= 0.5.0, < 2)
-    rdf (2.2.12)
+    rdf (3.0.2)
       hamster (~> 3.0)
       link_header (~> 0.0, >= 0.0.8)
-    rdf-normalize (0.3.2)
-      rdf (~> 2.0)
-    redis (3.3.5)
+    rdf-normalize (0.3.3)
+      rdf (>= 2.2, < 4.0)
+    redis (4.0.1)
     redis-actionpack (5.0.2)
       actionpack (>= 4.0, < 6)
       redis-rack (>= 1, < 3)
@@ -445,24 +479,25 @@ GEM
       redis-store (>= 1.3, < 2)
     redis-namespace (1.6.0)
       redis (>= 3.0.4)
-    redis-rack (2.0.3)
+    redis-rack (2.0.4)
       rack (>= 1.5, < 3)
       redis-store (>= 1.2, < 2)
     redis-rails (5.0.2)
       redis-actionpack (>= 5.0, < 6)
       redis-activesupport (>= 5.0, < 6)
       redis-store (>= 1.2, < 2)
-    redis-store (1.4.1)
+    redis-store (1.5.0)
       redis (>= 2.2, < 5)
-    request_store (1.3.2)
+    request_store (1.4.1)
+      rack (>= 1.4)
     responders (2.4.0)
       actionpack (>= 4.2.0, < 5.3)
       railties (>= 4.2.0, < 5.3)
     rotp (2.1.2)
-    rpam2 (3.1.0)
+    rpam2 (4.0.2)
     rqrcode (0.10.1)
       chunky_png (~> 1.0)
-    rspec-core (3.7.0)
+    rspec-core (3.7.1)
       rspec-support (~> 3.7.0)
     rspec-expectations (3.7.0)
       diff-lcs (>= 1.2.0, < 2.0)
@@ -481,42 +516,41 @@ GEM
     rspec-sidekiq (3.0.3)
       rspec-core (~> 3.0, >= 3.0.0)
       sidekiq (>= 2.4.0)
-    rspec-support (3.7.0)
-    rubocop (0.51.0)
+    rspec-support (3.7.1)
+    rubocop (0.55.0)
       parallel (~> 1.10)
-      parser (>= 2.3.3.1, < 3.0)
+      parser (>= 2.5)
       powerpack (~> 0.1)
-      rainbow (>= 2.2.2, < 3.0)
+      rainbow (>= 2.2.2, < 4.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (~> 1.0, >= 1.0.1)
-    ruby-oembed (0.12.0)
     ruby-progressbar (1.9.0)
     ruby-saml (1.7.2)
       nokogiri (>= 1.5.10)
     rufus-scheduler (3.4.2)
       et-orbi (~> 1.0)
     safe_yaml (1.0.4)
-    sanitize (4.5.0)
+    sanitize (4.6.4)
       crass (~> 1.0.2)
       nokogiri (>= 1.4.4)
-      nokogumbo (~> 1.4.1)
-    sass (3.5.3)
+      nokogumbo (~> 1.4)
+    sass (3.5.6)
       sass-listen (~> 4.0.0)
     sass-listen (4.0.0)
       rb-fsevent (~> 0.9, >= 0.9.4)
       rb-inotify (~> 0.9, >= 0.9.7)
-    scss_lint (0.56.0)
+    scss_lint (0.57.0)
       rake (>= 0.9, < 13)
-      sass (~> 3.5.3)
-    sidekiq (5.0.5)
+      sass (~> 3.5.5)
+    sidekiq (5.1.3)
       concurrent-ruby (~> 1.0)
       connection_pool (~> 2.2, >= 2.2.0)
       rack-protection (>= 1.5.0)
-      redis (>= 3.3.4, < 5)
+      redis (>= 3.3.5, < 5)
     sidekiq-bulk (0.1.1)
       activesupport
       sidekiq
-    sidekiq-scheduler (2.1.10)
+    sidekiq-scheduler (2.2.1)
       redis (>= 3, < 5)
       rufus-scheduler (~> 3.2)
       sidekiq (>= 3)
@@ -526,11 +560,11 @@ GEM
       thor (~> 0)
     simple-navigation (4.0.5)
       activesupport (>= 2.3.2)
-    simple_form (3.5.0)
-      actionpack (> 4, < 5.2)
-      activemodel (> 4, < 5.2)
-    simplecov (0.15.1)
-      docile (~> 1.1.0)
+    simple_form (4.0.0)
+      actionpack (> 4)
+      activemodel (> 4)
+    simplecov (0.16.1)
+      docile (~> 1.1)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
@@ -541,17 +575,21 @@ GEM
       actionpack (>= 4.0)
       activesupport (>= 4.0)
       sprockets (>= 3.0.0)
-    sshkit (1.15.1)
+    sshkit (1.16.0)
       net-scp (>= 1.1.2)
       net-ssh (>= 2.8.0)
+    stackprof (0.2.11)
     statsd-ruby (1.2.1)
+    stoplight (2.1.3)
     streamio-ffmpeg (3.0.2)
       multi_json (~> 1.8)
-    strong_migrations (0.1.9)
+    strong_migrations (0.2.2)
       activerecord (>= 3.2.0)
     temple (0.8.0)
     terminal-table (1.8.0)
       unicode-display_width (~> 1.1, >= 1.1.1)
+    terrapin (0.6.0)
+      climate_control (>= 0.0.3, < 1.0)
     thor (0.20.0)
     thread (0.2.2)
     thread_safe (0.3.6)
@@ -559,10 +597,10 @@ GEM
     timers (4.1.2)
       hitimes
     tty-color (0.4.2)
-    tty-command (0.7.0)
+    tty-command (0.8.0)
       pastel (~> 0.7.0)
     tty-cursor (0.5.0)
-    tty-prompt (0.15.0)
+    tty-prompt (0.16.0)
       necromancer (~> 0.4.0)
       pastel (~> 0.7.0)
       timers (~> 4.0)
@@ -575,34 +613,34 @@ GEM
     tty-screen (0.6.4)
     twitter-text (1.14.7)
       unf (~> 0.1.0)
-    tzinfo (1.2.4)
+    tzinfo (1.2.5)
       thread_safe (~> 0.1)
-    tzinfo-data (1.2017.3)
+    tzinfo-data (1.2018.4)
       tzinfo (>= 1.0.0)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.4)
-    unicode-display_width (1.3.0)
-    uniform_notifier (1.10.0)
+    unf_ext (0.0.7.5)
+    unicode-display_width (1.3.2)
+    uniform_notifier (1.11.0)
     warden (1.2.7)
       rack (>= 1.0)
-    webmock (3.1.1)
+    webmock (3.3.0)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff
-    webpacker (3.0.2)
+    webpacker (3.4.3)
       activesupport (>= 4.2)
       rack-proxy (>= 0.6.1)
       railties (>= 4.2)
     webpush (0.3.3)
       hkdf (~> 0.2)
       jwt (~> 2.0)
-    websocket-driver (0.6.5)
+    websocket-driver (0.7.0)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.3)
     wisper (2.0.0)
-    xpath (2.1.0)
-      nokogiri (~> 1.3)
+    xpath (3.0.0)
+      nokogiri (~> 1.8)
 
 PLATFORMS
   ruby
@@ -612,52 +650,54 @@ DEPENDENCIES
   active_record_query_trace (~> 1.5)
   addressable (~> 2.5)
   annotate (~> 2.7)
-  aws-sdk (~> 2.10)
+  aws-sdk-s3 (~> 1.9)
   better_errors (~> 2.4)
   binding_of_caller (~> 0.7)
-  bootsnap
-  brakeman (~> 4.0)
+  bootsnap (~> 1.3)
+  brakeman (~> 4.2)
   browser
-  bullet (~> 5.5)
+  bullet (~> 5.7)
   bundler-audit (~> 0.6)
   capistrano (~> 3.10)
   capistrano-rails (~> 1.3)
   capistrano-rbenv (~> 2.1)
   capistrano-yarn (~> 2.0)
-  capybara (~> 2.15)
-  charlock_holmes (~> 0.7.5)
+  capybara (~> 2.18)
+  charlock_holmes (~> 0.7.6)
   chewy (~> 5.0)
   cld3 (~> 3.2.0)
   climate_control (~> 0.2)
+  derailed_benchmarks
   devise (~> 4.4)
   devise-two-factor (~> 3.0)
-  devise_pam_authenticatable2 (~> 8.0)
-  doorkeeper (~> 4.2)
-  dotenv-rails (~> 2.2)
-  fabrication (~> 2.18)
-  faker (~> 1.7)
+  devise_pam_authenticatable2 (~> 9.1)
+  doorkeeper (~> 4.2, < 4.3)
+  dotenv-rails (~> 2.2, < 2.3)
+  fabrication (~> 2.20)
+  faker (~> 1.8)
   fast_blank (~> 1.0)
   fastimage
   fog-core (~> 1.45)
-  fog-local (~> 0.4)
+  fog-local (~> 0.5)
   fog-openstack (~> 0.1)
   fuubar (~> 2.2)
   goldfinger (~> 2.1)
   hamlit-rails (~> 0.2)
   hiredis (~> 0.6)
   htmlentities (~> 4.3)
-  http (~> 3.0)
+  http (~> 3.2)
   http_accept_language (~> 2.1)
-  httplog (~> 0.99)
+  http_parser.rb (~> 0.6)!
+  httplog (~> 1.0)
   i18n-tasks (~> 0.9)
   idn-ruby
   iso-639
-  json-ld-preloaded (~> 2.2.1)
+  json-ld (~> 2.2)
   kaminari (~> 1.1)
   letter_opener (~> 1.4)
   letter_opener_web (~> 1.3)
   link_header (~> 0.0)
-  lograge (~> 0.7)
+  lograge (~> 0.10)
   mario-redis-lock (~> 1.2)
   memory_profiler
   microformats (~> 4.0)
@@ -665,58 +705,61 @@ DEPENDENCIES
   net-ldap (~> 0.10)
   nokogiri (~> 1.8)
   nsa (~> 0.2)
-  oj (~> 3.3)
+  oj (~> 3.5)
   omniauth (~> 1.2)
   omniauth-cas (~> 1.1)
   omniauth-saml (~> 1.10)
   ostatus2 (~> 2.0)
-  ox (~> 2.8)
-  paperclip (~> 5.1)
+  ox (~> 2.9)
+  paperclip (~> 6.0)
   paperclip-av-transcoder (~> 0.6)
-  parallel_tests (~> 2.17)
-  pg (~> 0.20)
-  pghero (~> 1.7)
-  pkg-config (~> 1.2)
+  parallel_tests (~> 2.21)
+  pg (~> 1.0)
+  pghero (~> 2.1)
+  pkg-config (~> 1.3)
+  posix-spawn!
   premailer-rails
   private_address_check (~> 0.4.1)
+  pry-byebug (~> 3.6)
   pry-rails (~> 0.3)
-  puma (~> 3.10)
+  puma (~> 3.11)
   pundit (~> 1.1)
-  rack-attack (~> 5.0)
-  rack-cors (~> 0.4)
+  rack-attack (~> 5.2)
+  rack-cors (~> 1.0)
   rack-timeout (~> 0.4)
-  rails (~> 5.1.4)
+  rails (~> 5.2.0)
   rails-controller-testing (~> 1.0)
-  rails-i18n (~> 5.0)
+  rails-i18n (~> 5.1)
   rails-settings-cached (~> 0.6)
-  rdf-normalize (~> 0.3.1)
-  redis (~> 3.3)
+  rdf-normalize (~> 0.3)
+  redis (~> 4.0)
   redis-namespace (~> 1.5)
   redis-rails (~> 5.0)
   rqrcode (~> 0.10)
   rspec-rails (~> 3.7)
   rspec-sidekiq (~> 3.0)
-  rubocop
-  ruby-oembed (~> 0.12)
+  rubocop (~> 0.55)
   ruby-progressbar (~> 1.4)
-  sanitize (~> 4.4)
-  scss_lint (~> 0.55)
-  sidekiq (~> 5.0)
+  sanitize (~> 4.6)
+  scss_lint (~> 0.57)
+  sidekiq (~> 5.1)
   sidekiq-bulk (~> 0.1.1)
-  sidekiq-scheduler (~> 2.1)
+  sidekiq-scheduler (~> 2.2)
   sidekiq-unique-jobs (~> 5.0)
   simple-navigation (~> 4.0)
-  simple_form (~> 3.4)
-  simplecov (~> 0.14)
+  simple_form (~> 4.0)
+  simplecov (~> 0.16)
   sprockets-rails (~> 3.2)
+  stackprof
+  stoplight (~> 2.1.3)
   streamio-ffmpeg (~> 3.0)
-  strong_migrations
-  tty-command
-  tty-prompt
+  strong_migrations (~> 0.2)
+  tty-command (~> 0.8)
+  tty-prompt (~> 0.16)
   twitter-text (~> 1.14)
-  tzinfo-data (~> 1.2017)
-  webmock (~> 3.0)
-  webpacker (~> 3.0)
+  tzinfo-data (~> 1.2018)
+  webmock (~> 3.3)
+  webpacker (~> 3.4)
   webpush
 
 RUBY VERSION

README.md

@@ -1,10 +1,10 @@
 ![Mastodon](https://i.imgur.com/NhZc40l.png)
 ========
 
-[![Build Status](https://img.shields.io/travis/tootsuite/mastodon.svg)][travis]
+[![Build Status](https://img.shields.io/circleci/project/github/tootsuite/mastodon.svg)][circleci]
 [![Code Climate](https://img.shields.io/codeclimate/maintainability/tootsuite/mastodon.svg)][code_climate]
 
-[travis]: https://travis-ci.org/tootsuite/mastodon
+[circleci]: https://circleci.com/gh/tootsuite/mastodon
 [code_climate]: https://codeclimate.com/github/tootsuite/mastodon
 
 Mastodon is a **free, open-source social network server** based on **open web protocols** like ActivityPub and OStatus. The social focus of the project is a viable decentralized alternative to commercial social media silos that returns the control of the content distribution channels to the people. The technical focus of the project is a good user interface, a clean REST API for 3rd party apps and robust anti-abuse tools.

app/controllers/accounts_controller.rb

@@ -20,9 +20,10 @@ class AccountsController < ApplicationController
         @pinned_statuses = cache_collection(@account.pinned_statuses, Status) if show_pinned_statuses?
         @statuses        = filtered_status_page(params)
         @statuses        = cache_collection(@statuses, Status)
+
         unless @statuses.empty?
-          @older_url        = older_url if @statuses.last.id > filtered_statuses.last.id
-          @newer_url        = newer_url if @statuses.first.id < filtered_statuses.first.id
+          @older_url = older_url if @statuses.last.id > filtered_statuses.last.id
+          @newer_url = newer_url if @statuses.first.id < filtered_statuses.first.id
         end
       end
 
@@ -31,6 +32,11 @@ class AccountsController < ApplicationController
         render xml: OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.feed(@account, @entries.reject { |entry| entry.status.nil? }))
       end
 
+      format.rss do
+        @statuses = cache_collection(default_statuses.without_reblogs.without_replies.limit(PAGE_SIZE), Status)
+        render xml: RSS::AccountSerializer.render(@account, @statuses)
+      end
+
       format.json do
         skip_session!
 

app/controllers/activitypub/collections_controller.rb

@@ -22,7 +22,7 @@ class ActivityPub::CollectionsController < Api::BaseController
   end
 
   def set_statuses
-    @statuses = scope_for_collection.paginate_by_max_id(20, params[:max_id], params[:since_id])
+    @statuses = scope_for_collection
     @statuses = cache_collection(@statuses, Status)
   end
 

app/controllers/activitypub/outboxes_controller.rb

@@ -1,14 +1,14 @@
 # frozen_string_literal: true
 
 class ActivityPub::OutboxesController < Api::BaseController
+  LIMIT = 20
+
   include SignatureVerification
 
   before_action :set_account
+  before_action :set_statuses
 
   def show
-    @statuses = @account.statuses.permitted_for(@account, signed_request_account).paginate_by_max_id(20, params[:max_id], params[:since_id])
-    @statuses = cache_collection(@statuses, Status)
-
     render json: outbox_presenter, serializer: ActivityPub::OutboxSerializer, adapter: ActivityPub::Adapter, content_type: 'application/activity+json'
   end
 
@@ -19,11 +19,47 @@ class ActivityPub::OutboxesController < Api::BaseController
   end
 
   def outbox_presenter
-    ActivityPub::CollectionPresenter.new(
-      id: account_outbox_url(@account),
-      type: :ordered,
-      size: @account.statuses_count,
-      items: @statuses
-    )
+    if page_requested?
+      ActivityPub::CollectionPresenter.new(
+        id: account_outbox_url(@account, page_params),
+        type: :ordered,
+        part_of: account_outbox_url(@account),
+        prev: prev_page,
+        next: next_page,
+        items: @statuses
+      )
+    else
+      ActivityPub::CollectionPresenter.new(
+        id: account_outbox_url(@account),
+        type: :ordered,
+        size: @account.statuses_count,
+        first: account_outbox_url(@account, page: true),
+        last: account_outbox_url(@account, page: true, min_id: 0)
+      )
+    end
+  end
+
+  def next_page
+    account_outbox_url(@account, page: true, max_id: @statuses.last.id) if @statuses.size == LIMIT
+  end
+
+  def prev_page
+    account_outbox_url(@account, page: true, min_id: @statuses.first.id) unless @statuses.empty?
+  end
+
+  def set_statuses
+    return unless page_requested?
+
+    @statuses = @account.statuses.permitted_for(@account, signed_request_account)
+    @statuses = params[:min_id].present? ? @statuses.paginate_by_min_id(LIMIT, params[:min_id]).reverse : @statuses.paginate_by_max_id(LIMIT, params[:max_id])
+    @statuses = cache_collection(@statuses, Status)
+  end
+
+  def page_requested?
+    params[:page] == 'true'
+  end
+
+  def page_params
+    { page: true, max_id: params[:max_id], min_id: params[:min_id] }.compact
   end
 end

app/controllers/admin/accounts_controller.rb

@@ -2,7 +2,7 @@
 
 module Admin
   class AccountsController < BaseController
-    before_action :set_account, only: [:show, :subscribe, :unsubscribe, :redownload, :enable, :disable, :memorialize]
+    before_action :set_account, only: [:show, :subscribe, :unsubscribe, :redownload, :remove_avatar, :enable, :disable, :memorialize]
     before_action :require_remote_account!, only: [:subscribe, :unsubscribe, :redownload]
     before_action :require_local_account!, only: [:enable, :disable, :memorialize]
 
@@ -60,6 +60,17 @@ module Admin
       redirect_to admin_account_path(@account.id)
     end
 
+    def remove_avatar
+      authorize @account, :remove_avatar?
+
+      @account.avatar = nil
+      @account.save!
+
+      log_action :remove_avatar, @account.user
+
+      redirect_to admin_account_path(@account.id)
+    end
+
     private
 
     def set_account

app/controllers/admin/change_emails_controller.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Admin
+  class ChangeEmailsController < BaseController
+    before_action :set_account
+    before_action :require_local_account!
+
+    def show
+      authorize @user, :change_email?
+    end
+
+    def update
+      authorize @user, :change_email?
+
+      new_email = resource_params.fetch(:unconfirmed_email)
+
+      if new_email != @user.email
+        @user.update!(
+          unconfirmed_email: new_email,
+          # Regenerate the confirmation token:
+          confirmation_token: nil
+        )
+
+        log_action :change_email, @user
+
+        @user.send_confirmation_instructions
+      end
+
+      redirect_to admin_account_path(@account.id), notice: I18n.t('admin.accounts.change_email.changed_msg')
+    end
+
+    private
+
+    def set_account
+      @account = Account.find(params[:account_id])
+      @user = @account.user
+    end
+
+    def require_local_account!
+      redirect_to admin_account_path(@account.id) unless @account.local? && @account.user.present?
+    end
+
+    def resource_params
+      params.require(:user).permit(
+        :unconfirmed_email
+      )
+    end
+  end
+end

app/controllers/admin/confirmations_controller.rb

@@ -3,6 +3,7 @@
 module Admin
   class ConfirmationsController < BaseController
     before_action :set_user
+    before_action :check_confirmation, only: [:resend]
 
     def create
       authorize @user, :confirm?
@@ -11,10 +12,28 @@ module Admin
       redirect_to admin_accounts_path
     end
 
+    def resend
+      authorize @user, :confirm?
+
+      @user.resend_confirmation_instructions
+
+      log_action :confirm, @user
+
+      flash[:notice] = I18n.t('admin.accounts.resend_confirmation.success')
+      redirect_to admin_accounts_path
+    end
+
     private
 
     def set_user
       @user = Account.find(params[:account_id]).user || raise(ActiveRecord::RecordNotFound)
     end
+
+    def check_confirmation
+      if @user.confirmed?
+        flash[:error] = I18n.t('admin.accounts.resend_confirmation.already_confirmed')
+        redirect_to admin_accounts_path
+      end
+    end
   end
 end

app/controllers/admin/report_notes_controller.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Admin
+  class ReportNotesController < BaseController
+    before_action :set_report_note, only: [:destroy]
+
+    def create
+      authorize ReportNote, :create?
+
+      @report_note = current_account.report_notes.new(resource_params)
+      @report = @report_note.report
+
+      if @report_note.save
+        if params[:create_and_resolve]
+          @report.resolve!(current_account)
+          log_action :resolve, @report
+
+          redirect_to admin_reports_path, notice: I18n.t('admin.reports.resolved_msg')
+          return
+        end
+
+        if params[:create_and_unresolve]
+          @report.unresolve!
+          log_action :reopen, @report
+        end
+
+        redirect_to admin_report_path(@report), notice: I18n.t('admin.report_notes.created_msg')
+      else
+        @report_notes = @report.notes.latest
+        @report_history = @report.history
+        @form = Form::StatusBatch.new
+
+        render template: 'admin/reports/show'
+      end
+    end
+
+    def destroy
+      authorize @report_note, :destroy?
+      @report_note.destroy!
+      redirect_to admin_report_path(@report_note.report_id), notice: I18n.t('admin.report_notes.destroyed_msg')
+    end
+
+    private
+
+    def resource_params
+      params.require(:report_note).permit(
+        :content,
+        :report_id
+      )
+    end
+
+    def set_report_note
+      @report_note = ReportNote.find(params[:id])
+    end
+  end
+end

app/controllers/admin/reported_statuses_controller.rb

@@ -3,31 +3,16 @@
 module Admin
   class ReportedStatusesController < BaseController
     before_action :set_report
-    before_action :set_status, only: [:update, :destroy]
 
     def create
       authorize :status, :update?
 
-      @form         = Form::StatusBatch.new(form_status_batch_params.merge(current_account: current_account))
+      @form         = Form::StatusBatch.new(form_status_batch_params.merge(current_account: current_account, action: action_from_button))
       flash[:alert] = I18n.t('admin.statuses.failed_to_execute') unless @form.save
 
       redirect_to admin_report_path(@report)
     end
 
-    def update
-      authorize @status, :update?
-      @status.update!(status_params)
-      log_action :update, @status
-      redirect_to admin_report_path(@report)
-    end
-
-    def destroy
-      authorize @status, :destroy?
-      RemovalWorker.perform_async(@status.id)
-      log_action :destroy, @status
-      render json: @status
-    end
-
     private
 
     def status_params
@@ -35,15 +20,21 @@ module Admin
     end
 
     def form_status_batch_params
-      params.require(:form_status_batch).permit(:action, status_ids: [])
+      params.require(:form_status_batch).permit(status_ids: [])
     end
 
-    def set_report
-      @report = Report.find(params[:report_id])
+    def action_from_button
+      if params[:nsfw_on]
+        'nsfw_on'
+      elsif params[:nsfw_off]
+        'nsfw_off'
+      elsif params[:delete]
+        'delete'
+      end
     end
 
-    def set_status
-      @status = @report.statuses.find(params[:id])
+    def set_report
+      @report = Report.find(params[:report_id])
     end
   end
 end

app/controllers/admin/reports_controller.rb

@@ -11,45 +11,61 @@ module Admin
 
     def show
       authorize @report, :show?
-      @form = Form::StatusBatch.new
+
+      @report_note  = @report.notes.new
+      @report_notes = (@report.notes.latest + @report.history).sort_by(&:created_at)
+      @form         = Form::StatusBatch.new
     end
 
     def update
       authorize @report, :update?
       process_report
-      redirect_to admin_report_path(@report)
+
+      if @report.action_taken?
+        redirect_to admin_reports_path, notice: I18n.t('admin.reports.resolved_msg')
+      else
+        redirect_to admin_report_path(@report)
+      end
     end
 
     private
 
     def process_report
       case params[:outcome].to_s
+      when 'assign_to_self'
+        @report.update!(assigned_account_id: current_account.id)
+        log_action :assigned_to_self, @report
+      when 'unassign'
+        @report.update!(assigned_account_id: nil)
+        log_action :unassigned, @report
+      when 'reopen'
+        @report.unresolve!
+        log_action :reopen, @report
       when 'resolve'
-        @report.update!(action_taken_by_current_attributes)
+        @report.resolve!(current_account)
         log_action :resolve, @report
       when 'suspend'
         Admin::SuspensionWorker.perform_async(@report.target_account.id)
+
         log_action :resolve, @report
         log_action :suspend, @report.target_account
+
         resolve_all_target_account_reports
       when 'silence'
         @report.target_account.update!(silenced: true)
+
         log_action :resolve, @report
         log_action :silence, @report.target_account
+
         resolve_all_target_account_reports
       else
         raise ActiveRecord::RecordNotFound
       end
-    end
-
-    def action_taken_by_current_attributes
-      { action_taken: true, action_taken_by_account_id: current_account.id }
+      @report.reload
     end
 
     def resolve_all_target_account_reports
-      unresolved_reports_for_target_account.update_all(
-        action_taken_by_current_attributes
-      )
+      unresolved_reports_for_target_account.update_all(action_taken: true, action_taken_by_account_id: current_account.id)
     end
 
     def unresolved_reports_for_target_account

app/controllers/admin/statuses_controller.rb

@@ -5,14 +5,13 @@ module Admin
     helper_method :current_params
 
     before_action :set_account
-    before_action :set_status, only: [:update, :destroy]
 
     PER_PAGE = 20
 
     def index
       authorize :status, :index?
 
-      @statuses = @account.statuses
+      @statuses = @account.statuses.where(visibility: [:public, :unlisted])
 
       if params[:media]
         account_media_status_ids = @account.media_attachments.attached.reorder(nil).select(:status_id).distinct
@@ -26,40 +25,18 @@ module Admin
     def create
       authorize :status, :update?
 
-      @form         = Form::StatusBatch.new(form_status_batch_params.merge(current_account: current_account))
+      @form         = Form::StatusBatch.new(form_status_batch_params.merge(current_account: current_account, action: action_from_button))
       flash[:alert] = I18n.t('admin.statuses.failed_to_execute') unless @form.save
 
       redirect_to admin_account_statuses_path(@account.id, current_params)
     end
 
-    def update
-      authorize @status, :update?
-      @status.update!(status_params)
-      log_action :update, @status
-      redirect_to admin_account_statuses_path(@account.id, current_params)
-    end
-
-    def destroy
-      authorize @status, :destroy?
-      RemovalWorker.perform_async(@status.id)
-      log_action :destroy, @status
-      render json: @status
-    end
-
     private
 
-    def status_params
-      params.require(:status).permit(:sensitive)
-    end
-
     def form_status_batch_params
       params.require(:form_status_batch).permit(:action, status_ids: [])
     end
 
-    def set_status
-      @status = @account.statuses.find(params[:id])
-    end
-
     def set_account
       @account = Account.find(params[:account_id])
     end
@@ -72,5 +49,15 @@ module Admin
         page: page > 1 && page,
       }.select { |_, value| value.present? }
     end
+
+    def action_from_button
+      if params[:nsfw_on]
+        'nsfw_on'
+      elsif params[:nsfw_off]
+        'nsfw_off'
+      elsif params[:delete]
+        'delete'
+      end
+    end
   end
 end

app/controllers/api/base_controller.rb

@@ -66,8 +66,10 @@ class Api::BaseController < ApplicationController
   end
 
   def require_user!
-    if current_user
+    if current_user && !current_user.disabled?
       set_user_activity
+    elsif current_user
+      render json: { error: 'Your login is currently disabled' }, status: 403
     else
       render json: { error: 'This method requires an authenticated user' }, status: 422
     end

app/controllers/api/v1/accounts/credentials_controller.rb

@@ -13,6 +13,7 @@ class Api::V1::Accounts::CredentialsController < Api::BaseController
   def update
     @account = current_account
     UpdateAccountService.new.call(@account, account_params, raise_error: true)
+    UserSettingsDecorator.new(current_user).update(user_settings_params) if user_settings_params
     ActivityPub::UpdateDistributionWorker.perform_async(@account.id)
     render json: @account, serializer: REST::CredentialAccountSerializer
   end
@@ -20,6 +21,17 @@ class Api::V1::Accounts::CredentialsController < Api::BaseController
   private
 
   def account_params
-    params.permit(:display_name, :note, :avatar, :header, :locked)
+    params.permit(:display_name, :note, :avatar, :header, :locked, :bot, fields_attributes: [:name, :value])
+  end
+
+  def user_settings_params
+    return nil unless params.key?(:source)
+
+    source_params = params.require(:source)
+
+    {
+      'setting_default_privacy' => source_params.fetch(:privacy, @account.user.setting_default_privacy),
+      'setting_default_sensitive' => source_params.fetch(:sensitive, @account.user.setting_default_sensitive),
+    }
   end
 end

app/controllers/api/v1/accounts/follower_accounts_controller.rb

@@ -19,6 +19,8 @@ class Api::V1::Accounts::FollowerAccountsController < Api::BaseController
   end
 
   def load_accounts
+    return [] if @account.user_hides_network? && current_account.id != @account.id
+
     default_accounts.merge(paginated_follows).to_a
   end
 
@@ -63,6 +65,6 @@ class Api::V1::Accounts::FollowerAccountsController < Api::BaseController
   end
 
   def pagination_params(core_params)
-    params.permit(:limit).merge(core_params)
+    params.slice(:limit).permit(:limit).merge(core_params)
   end
 end

app/controllers/api/v1/accounts/following_accounts_controller.rb

@@ -19,6 +19,8 @@ class Api::V1::Accounts::FollowingAccountsController < Api::BaseController
   end
 
   def load_accounts
+    return [] if @account.user_hides_network? && current_account.id != @account.id
+
     default_accounts.merge(paginated_follows).to_a
   end
 
@@ -63,6 +65,6 @@ class Api::V1::Accounts::FollowingAccountsController < Api::BaseController
   end
 
   def pagination_params(core_params)
-    params.permit(:limit).merge(core_params)
+    params.slice(:limit).permit(:limit).merge(core_params)
   end
 end

app/controllers/api/v1/accounts/statuses_controller.rb

@@ -27,19 +27,17 @@ class Api::V1::Accounts::StatusesController < Api::BaseController
   end
 
   def account_statuses
-    default_statuses.tap do |statuses|
-      statuses.merge!(only_media_scope) if truthy_param?(:only_media)
-      statuses.merge!(pinned_scope) if truthy_param?(:pinned)
-      statuses.merge!(no_replies_scope) if truthy_param?(:exclude_replies)
-    end
-  end
-
-  def default_statuses
-    permitted_account_statuses.paginate_by_max_id(
+    statuses = truthy_param?(:pinned) ? pinned_scope : permitted_account_statuses
+    statuses = statuses.paginate_by_max_id(
       limit_param(DEFAULT_STATUSES_LIMIT),
       params[:max_id],
       params[:since_id]
     )
+
+    statuses.merge!(only_media_scope) if truthy_param?(:only_media)
+    statuses.merge!(no_replies_scope) if truthy_param?(:exclude_replies)
+
+    statuses
   end
 
   def permitted_account_statuses
@@ -69,7 +67,7 @@ class Api::V1::Accounts::StatusesController < Api::BaseController
   end
 
   def pagination_params(core_params)
-    params.permit(:limit, :only_media, :exclude_replies).merge(core_params)
+    params.slice(:limit, :only_media, :exclude_replies).permit(:limit, :only_media, :exclude_replies).merge(core_params)
   end
 
   def insert_pagination_headers

app/controllers/api/v1/accounts_controller.rb

@@ -5,6 +5,7 @@ class Api::V1::AccountsController < Api::BaseController
   before_action -> { doorkeeper_authorize! :follow }, only: [:follow, :unfollow, :block, :unblock, :mute, :unmute]
   before_action :require_user!, except: [:show]
   before_action :set_account
+  before_action :check_account_suspension, only: [:show]
 
   respond_to :json
 
@@ -54,4 +55,8 @@ class Api::V1::AccountsController < Api::BaseController
   def relationships(**options)
     AccountRelationshipsPresenter.new([@account.id], current_user.account_id, options)
   end
+
+  def check_account_suspension
+    gone if @account.suspended?
+  end
 end

app/controllers/api/v1/blocks_controller.rb

@@ -57,6 +57,6 @@ class Api::V1::BlocksController < Api::BaseController
   end
 
   def pagination_params(core_params)
-    params.permit(:limit).merge(core_params)
+    params.slice(:limit).permit(:limit).merge(core_params)
   end
 end

app/controllers/api/v1/domain_blocks_controller.rb

@@ -67,7 +67,7 @@ class Api::V1::DomainBlocksController < Api::BaseController
   end
 
   def pagination_params(core_params)
-    params.permit(:limit).merge(core_params)
+    params.slice(:limit).permit(:limit).merge(core_params)
   end
 
   def domain_block_params

app/controllers/api/v1/favourites_controller.rb

@@ -66,6 +66,6 @@ class Api::V1::FavouritesController < Api::BaseController
   end
 
   def pagination_params(core_params)
-    params.permit(:limit).merge(core_params)
+    params.slice(:limit).permit(:limit).merge(core_params)
   end
 end

app/controllers/api/v1/follow_requests_controller.rb

@@ -71,6 +71,6 @@ class Api::V1::FollowRequestsController < Api::BaseController
   end
 
   def pagination_params(core_params)
-    params.permit(:limit).merge(core_params)
+    params.slice(:limit).permit(:limit).merge(core_params)
   end
 end

app/controllers/api/v1/lists/accounts_controller.rb

@@ -88,7 +88,7 @@ class Api::V1::Lists::AccountsController < Api::BaseController
   end
 
   def pagination_params(core_params)
-    params.permit(:limit).merge(core_params)
+    params.slice(:limit).permit(:limit).merge(core_params)
   end
 
   def unlimited?

app/controllers/api/v1/mutes_controller.rb

@@ -59,6 +59,6 @@ class Api::V1::MutesController < Api::BaseController
   end
 
   def pagination_params(core_params)
-    params.permit(:limit).merge(core_params)
+    params.slice(:limit).permit(:limit).merge(core_params)
   end
 end

app/controllers/api/v1/notifications_controller.rb

@@ -82,6 +82,6 @@ class Api::V1::NotificationsController < Api::BaseController
   end
 
   def pagination_params(core_params)
-    params.permit(:limit, exclude_types: []).merge(core_params)
+    params.slice(:limit, :exclude_types).permit(:limit, exclude_types: []).merge(core_params)
   end
 end

app/controllers/api/v1/push/subscriptions_controller.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+class Api::V1::Push::SubscriptionsController < Api::BaseController
+  before_action -> { doorkeeper_authorize! :push }
+  before_action :require_user!
+  before_action :set_web_push_subscription
+
+  def create
+    @web_subscription&.destroy!
+
+    @web_subscription = ::Web::PushSubscription.create!(
+      endpoint: subscription_params[:endpoint],
+      key_p256dh: subscription_params[:keys][:p256dh],
+      key_auth: subscription_params[:keys][:auth],
+      data: data_params,
+      user_id: current_user.id,
+      access_token_id: doorkeeper_token.id
+    )
+
+    render json: @web_subscription, serializer: REST::WebPushSubscriptionSerializer
+  end
+
+  def show
+    raise ActiveRecord::RecordNotFound if @web_subscription.nil?
+
+    render json: @web_subscription, serializer: REST::WebPushSubscriptionSerializer
+  end
+
+  def update
+    raise ActiveRecord::RecordNotFound if @web_subscription.nil?
+
+    @web_subscription.update!(data: data_params)
+
+    render json: @web_subscription, serializer: REST::WebPushSubscriptionSerializer
+  end
+
+  def destroy
+    @web_subscription&.destroy!
+    render_empty
+  end
+
+  private
+
+  def set_web_push_subscription
+    @web_subscription = ::Web::PushSubscription.find_by(access_token_id: doorkeeper_token.id)
+  end
+
+  def subscription_params
+    params.require(:subscription).permit(:endpoint, keys: [:auth, :p256dh])
+  end
+
+  def data_params
+    return {} if params[:data].blank?
+    params.require(:data).permit(alerts: [:follow, :favourite, :reblog, :mention])
+  end
+end

app/controllers/api/v1/statuses/favourited_by_accounts_controller.rb

@@ -77,6 +77,6 @@ class Api::V1::Statuses::FavouritedByAccountsController < Api::BaseController
   end
 
   def pagination_params(core_params)
-    params.permit(:limit).merge(core_params)
+    params.slice(:limit).permit(:limit).merge(core_params)
   end
 end

app/controllers/api/v1/statuses/pins_controller.rb

@@ -39,7 +39,7 @@ class Api::V1::Statuses::PinsController < Api::BaseController
       adapter: ActivityPub::Adapter
     ).as_json
 
-    ActivityPub::RawDistributionWorker.perform_async(Oj.dump(json), current_account)
+    ActivityPub::RawDistributionWorker.perform_async(Oj.dump(json), current_account.id)
   end
 
   def distribute_remove_activity!
@@ -49,6 +49,6 @@ class Api::V1::Statuses::PinsController < Api::BaseController
       adapter: ActivityPub::Adapter
     ).as_json
 
-    ActivityPub::RawDistributionWorker.perform_async(Oj.dump(json), current_account)
+    ActivityPub::RawDistributionWorker.perform_async(Oj.dump(json), current_account.id)
   end
 end

app/controllers/api/v1/statuses/reblogged_by_accounts_controller.rb

@@ -74,6 +74,6 @@ class Api::V1::Statuses::RebloggedByAccountsController < Api::BaseController
   end
 
   def pagination_params(core_params)
-    params.permit(:limit).merge(core_params)
+    params.slice(:limit).permit(:limit).merge(core_params)
   end
 end

app/controllers/api/v1/statuses_controller.rb

@@ -10,6 +10,12 @@ class Api::V1::StatusesController < Api::BaseController
 
   respond_to :json
 
+  # This API was originally unlimited, pagination cannot be introduced without
+  # breaking backwards-compatibility. Arbitrarily high number to cover most
+  # conversations as quasi-unlimited, it would be too much work to render more
+  # than this anyway
+  CONTEXT_LIMIT = 4_096
+
   def show
     cached  = Rails.cache.read(@status.cache_key)
     @status = cached unless cached.nil?
@@ -17,8 +23,8 @@ class Api::V1::StatusesController < Api::BaseController
   end
 
   def context
-    ancestors_results   = @status.in_reply_to_id.nil? ? [] : @status.ancestors(current_account)
-    descendants_results = @status.descendants(current_account)
+    ancestors_results   = @status.in_reply_to_id.nil? ? [] : @status.ancestors(CONTEXT_LIMIT, current_account)
+    descendants_results = @status.descendants(CONTEXT_LIMIT, current_account)
     loaded_ancestors    = cache_collection(ancestors_results, Status)
     loaded_descendants  = cache_collection(descendants_results, Status)
 
@@ -76,7 +82,7 @@ class Api::V1::StatusesController < Api::BaseController
   end
 
   def pagination_params(core_params)
-    params.permit(:limit).merge(core_params)
+    params.slice(:limit).permit(:limit).merge(core_params)
   end
 
   def authorize_if_got_token

app/controllers/api/v1/timelines/direct_controller.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+class Api::V1::Timelines::DirectController < Api::BaseController
+  before_action -> { doorkeeper_authorize! :read }, only: [:show]
+  before_action :require_user!, only: [:show]
+  after_action :insert_pagination_headers, unless: -> { @statuses.empty? }
+
+  respond_to :json
+
+  def show
+    @statuses = load_statuses
+    render json: @statuses, each_serializer: REST::StatusSerializer, relationships: StatusRelationshipsPresenter.new(@statuses, current_user&.account_id)
+  end
+
+  private
+
+  def load_statuses
+    cached_direct_statuses
+  end
+
+  def cached_direct_statuses
+    cache_collection direct_statuses, Status
+  end
+
+  def direct_statuses
+    direct_timeline_statuses.paginate_by_max_id(
+      limit_param(DEFAULT_STATUSES_LIMIT),
+      params[:max_id],
+      params[:since_id]
+    )
+  end
+
+  def direct_timeline_statuses
+    Status.as_direct_timeline(current_account)
+  end
+
+  def insert_pagination_headers
+    set_pagination_headers(next_path, prev_path)
+  end
+
+  def pagination_params(core_params)
+    params.permit(:local, :limit).merge(core_params)
+  end
+
+  def next_path
+    api_v1_timelines_direct_url pagination_params(max_id: pagination_max_id)
+  end
+
+  def prev_path
+    api_v1_timelines_direct_url pagination_params(since_id: pagination_since_id)
+  end
+
+  def pagination_max_id
+    @statuses.last.id
+  end
+
+  def pagination_since_id
+    @statuses.first.id
+  end
+end

app/controllers/api/v1/timelines/home_controller.rb

@@ -43,7 +43,7 @@ class Api::V1::Timelines::HomeController < Api::BaseController
   end
 
   def pagination_params(core_params)
-    params.permit(:local, :limit).merge(core_params)
+    params.slice(:local, :limit).permit(:local, :limit).merge(core_params)
   end
 
   def next_path

app/controllers/api/v1/timelines/list_controller.rb

@@ -45,7 +45,7 @@ class Api::V1::Timelines::ListController < Api::BaseController
   end
 
   def pagination_params(core_params)
-    params.permit(:limit).merge(core_params)
+    params.slice(:limit).permit(:limit).merge(core_params)
   end
 
   def next_path

app/controllers/api/v1/timelines/public_controller.rb

@@ -45,7 +45,7 @@ class Api::V1::Timelines::PublicController < Api::BaseController
   end
 
   def pagination_params(core_params)
-    params.permit(:local, :limit, :only_media).merge(core_params)
+    params.slice(:local, :limit, :only_media).permit(:local, :limit, :only_media).merge(core_params)
   end
 
   def next_path

app/controllers/api/v1/timelines/tag_controller.rb

@@ -54,7 +54,7 @@ class Api::V1::Timelines::TagController < Api::BaseController
   end
 
   def pagination_params(core_params)
-    params.permit(:local, :limit, :only_media).merge(core_params)
+    params.slice(:local, :limit, :only_media).permit(:local, :limit, :only_media).merge(core_params)
   end
 
   def next_path

app/controllers/api/web/base_controller.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class Api::Web::BaseController < Api::BaseController
+  protect_from_forgery with: :exception
+
+  rescue_from ActionController::InvalidAuthenticityToken do
+    render json: { error: "Can't verify CSRF token authenticity." }, status: 422
+  end
+end

app/controllers/api/web/embeds_controller.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class Api::Web::EmbedsController < Api::BaseController
+class Api::Web::EmbedsController < Api::Web::BaseController
   respond_to :json
 
   before_action :require_user!
@@ -9,9 +9,12 @@ class Api::Web::EmbedsController < Api::BaseController
     status = StatusFinder.new(params[:url]).status
     render json: status, serializer: OEmbedSerializer, width: 400
   rescue ActiveRecord::RecordNotFound
-    oembed = OEmbed::Providers.get(params[:url])
-    render json: Oj.dump(oembed.fields)
-  rescue OEmbed::NotFound
-    render json: {}, status: :not_found
+    oembed = FetchOEmbedService.new.call(params[:url])
+
+    if oembed
+      render json: oembed
+    else
+      render json: {}, status: :not_found
+    end
   end
 end

app/controllers/api/web/push_subscriptions_controller.rb

@@ -1,15 +1,11 @@
 # frozen_string_literal: true
 
-class Api::Web::PushSubscriptionsController < Api::BaseController
+class Api::Web::PushSubscriptionsController < Api::Web::BaseController
   respond_to :json
 
   before_action :require_user!
-  protect_from_forgery with: :exception
 
   def create
-    params.require(:subscription).require(:endpoint)
-    params.require(:subscription).require(:keys).require([:auth, :p256dh])
-
     active_session = current_session
 
     unless active_session.web_push_subscription.nil?
@@ -29,27 +25,38 @@ class Api::Web::PushSubscriptionsController < Api::BaseController
       },
     }
 
-    data.deep_merge!(params[:data]) if params[:data]
+    data.deep_merge!(data_params) if params[:data]
 
     web_subscription = ::Web::PushSubscription.create!(
-      endpoint: params[:subscription][:endpoint],
-      key_p256dh: params[:subscription][:keys][:p256dh],
-      key_auth: params[:subscription][:keys][:auth],
-      data: data
+      endpoint: subscription_params[:endpoint],
+      key_p256dh: subscription_params[:keys][:p256dh],
+      key_auth: subscription_params[:keys][:auth],
+      data: data,
+      user_id: active_session.user_id,
+      access_token_id: active_session.access_token_id
     )
 
     active_session.update!(web_push_subscription: web_subscription)
 
-    render json: web_subscription.as_payload
+    render json: web_subscription, serializer: REST::WebPushSubscriptionSerializer
   end
 
   def update
-    params.require([:id, :data])
+    params.require([:id])
 
     web_subscription = ::Web::PushSubscription.find(params[:id])
+    web_subscription.update!(data: data_params)
+
+    render json: web_subscription, serializer: REST::WebPushSubscriptionSerializer
+  end
 
-    web_subscription.update!(data: params[:data])
+  private
+
+  def subscription_params
+    @subscription_params ||= params.require(:subscription).permit(:endpoint, keys: [:auth, :p256dh])
+  end
 
-    render json: web_subscription.as_payload
+  def data_params
+    @data_params ||= params.require(:data).permit(alerts: [:follow, :favourite, :reblog, :mention])
   end
 end

app/controllers/api/web/settings_controller.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class Api::Web::SettingsController < Api::BaseController
+class Api::Web::SettingsController < Api::Web::BaseController
   respond_to :json
 
   before_action :require_user!

app/controllers/application_controller.rb

@@ -9,6 +9,7 @@ class ApplicationController < ActionController::Base
 
   include Localized
   include UserTrackingConcern
+  include SessionTrackingConcern
 
   helper_method :current_account
   helper_method :current_session
@@ -39,11 +40,11 @@ class ApplicationController < ActionController::Base
   end
 
   def require_admin!
-    redirect_to root_path unless current_user&.admin?
+    forbidden unless current_user&.admin?
   end
 
   def require_staff!
-    redirect_to root_path unless current_user&.staff?
+    forbidden unless current_user&.staff?
   end
 
   def check_suspension

app/controllers/concerns/localized.rb

@@ -29,10 +29,14 @@ module Localized
   end
 
   def preferred_locale
-    http_accept_language.preferred_language_from(I18n.available_locales)
+    http_accept_language.preferred_language_from(available_locales)
   end
 
   def compatible_locale
-    http_accept_language.compatible_language_from(I18n.available_locales)
+    http_accept_language.compatible_language_from(available_locales)
+  end
+
+  def available_locales
+    I18n.available_locales.reverse
   end
 end

app/controllers/concerns/remote_account_controller_concern.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module RemoteAccountControllerConcern
+  extend ActiveSupport::Concern
+
+  included do
+    layout 'public'
+    before_action :set_account
+    before_action :check_account_suspension
+  end
+
+  private
+
+  def set_account
+    @account = Account.find_remote!(params[:acct])
+  end
+
+  def check_account_suspension
+    gone if @account.suspended?
+  end
+end

app/controllers/concerns/session_tracking_concern.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module SessionTrackingConcern
+  extend ActiveSupport::Concern
+
+  UPDATE_SIGN_IN_HOURS = 24
+
+  included do
+    before_action :set_session_activity
+  end
+
+  private
+
+  def set_session_activity
+    return unless session_needs_update?
+    current_session.touch
+  end
+
+  def session_needs_update?
+    !current_session.nil? && current_session.updated_at < UPDATE_SIGN_IN_HOURS.hours.ago
+  end
+end

app/controllers/concerns/signature_verification.rb

@@ -107,9 +107,7 @@ module SignatureVerification
 
   def incompatible_signature?(signature_params)
     signature_params['keyId'].blank? ||
-      signature_params['signature'].blank? ||
-      signature_params['algorithm'].blank? ||
-      signature_params['algorithm'] != 'rsa-sha256'
+      signature_params['signature'].blank?
   end
 
   def account_from_key_id(key_id)

app/controllers/follower_accounts_controller.rb

@@ -4,14 +4,17 @@ class FollowerAccountsController < ApplicationController
   include AccountControllerConcern
 
   def index
-    @follows = Follow.where(target_account: @account).recent.page(params[:page]).per(FOLLOW_PER_PAGE).preload(:account)
-
     respond_to do |format|
       format.html do
-        @relationships = AccountRelationshipsPresenter.new(@follows.map(&:account_id), current_user.account_id) if user_signed_in?
+        next if @account.user_hides_network?
+
+        follows
+        @relationships = AccountRelationshipsPresenter.new(follows.map(&:account_id), current_user.account_id) if user_signed_in?
       end
 
       format.json do
+        raise Mastodon::NotPermittedError if params[:page].present? && @account.user_hides_network?
+
         render json: collection_presenter,
                serializer: ActivityPub::CollectionSerializer,
                adapter: ActivityPub::Adapter,
@@ -22,28 +25,31 @@ class FollowerAccountsController < ApplicationController
 
   private
 
+  def follows
+    @follows ||= Follow.where(target_account: @account).recent.page(params[:page]).per(FOLLOW_PER_PAGE).preload(:account)
+  end
+
   def page_url(page)
     account_followers_url(@account, page: page) unless page.nil?
   end
 
   def collection_presenter
-    page = ActivityPub::CollectionPresenter.new(
-      id: account_followers_url(@account, page: params.fetch(:page, 1)),
-      type: :ordered,
-      size: @account.followers_count,
-      items: @follows.map { |f| ActivityPub::TagManager.instance.uri_for(f.account) },
-      part_of: account_followers_url(@account),
-      next: page_url(@follows.next_page),
-      prev: page_url(@follows.prev_page)
-    )
     if params[:page].present?
-      page
+      ActivityPub::CollectionPresenter.new(
+        id: account_followers_url(@account, page: params.fetch(:page, 1)),
+        type: :ordered,
+        size: @account.followers_count,
+        items: follows.map { |f| ActivityPub::TagManager.instance.uri_for(f.account) },
+        part_of: account_followers_url(@account),
+        next: page_url(follows.next_page),
+        prev: page_url(follows.prev_page)
+      )
     else
       ActivityPub::CollectionPresenter.new(
         id: account_followers_url(@account),
         type: :ordered,
         size: @account.followers_count,
-        first: page
+        first: page_url(1)
       )
     end
   end

app/controllers/following_accounts_controller.rb

@@ -4,14 +4,17 @@ class FollowingAccountsController < ApplicationController
   include AccountControllerConcern
 
   def index
-    @follows = Follow.where(account: @account).recent.page(params[:page]).per(FOLLOW_PER_PAGE).preload(:target_account)
-
     respond_to do |format|
       format.html do
-        @relationships = AccountRelationshipsPresenter.new(@follows.map(&:target_account_id), current_user.account_id) if user_signed_in?
+        next if @account.user_hides_network?
+
+        follows
+        @relationships = AccountRelationshipsPresenter.new(follows.map(&:target_account_id), current_user.account_id) if user_signed_in?
       end
 
       format.json do
+        raise Mastodon::NotPermittedError if params[:page].present? && @account.user_hides_network?
+
         render json: collection_presenter,
                serializer: ActivityPub::CollectionSerializer,
                adapter: ActivityPub::Adapter,
@@ -22,28 +25,31 @@ class FollowingAccountsController < ApplicationController
 
   private
 
+  def follows
+    @follows ||= Follow.where(account: @account).recent.page(params[:page]).per(FOLLOW_PER_PAGE).preload(:target_account)
+  end
+
   def page_url(page)
     account_following_index_url(@account, page: page) unless page.nil?
   end
 
   def collection_presenter
-    page = ActivityPub::CollectionPresenter.new(
-      id: account_following_index_url(@account, page: params.fetch(:page, 1)),
-      type: :ordered,
-      size: @account.following_count,
-      items: @follows.map { |f| ActivityPub::TagManager.instance.uri_for(f.target_account) },
-      part_of: account_following_index_url(@account),
-      next: page_url(@follows.next_page),
-      prev: page_url(@follows.prev_page)
-    )
     if params[:page].present?
-      page
+      ActivityPub::CollectionPresenter.new(
+        id: account_following_index_url(@account, page: params.fetch(:page, 1)),
+        type: :ordered,
+        size: @account.following_count,
+        items: follows.map { |f| ActivityPub::TagManager.instance.uri_for(f.target_account) },
+        part_of: account_following_index_url(@account),
+        next: page_url(follows.next_page),
+        prev: page_url(follows.prev_page)
+      )
     else
       ActivityPub::CollectionPresenter.new(
         id: account_following_index_url(@account),
         type: :ordered,
         size: @account.following_count,
-        first: page
+        first: page_url(1)
       )
     end
   end

app/controllers/home_controller.rb

@@ -2,6 +2,7 @@
 
 class HomeController < ApplicationController
   before_action :authenticate_user!
+  before_action :set_referrer_policy_header
   before_action :set_initial_state_json
 
   def index
@@ -62,4 +63,8 @@ class HomeController < ApplicationController
       about_path
     end
   end
+
+  def set_referrer_policy_header
+    response.headers['Referrer-Policy'] = 'origin'
+  end
 end

app/controllers/invites_controller.rb

@@ -10,7 +10,7 @@ class InvitesController < ApplicationController
   def index
     authorize :invite, :create?
 
-    @invites = Invite.where(user: current_user)
+    @invites = invites
     @invite  = Invite.new(expires_in: 1.day.to_i)
   end
 
@@ -23,13 +23,13 @@ class InvitesController < ApplicationController
     if @invite.save
       redirect_to invites_path
     else
-      @invites = Invite.where(user: current_user)
+      @invites = invites
       render :index
     end
   end
 
   def destroy
-    @invite = Invite.where(user: current_user).find(params[:id])
+    @invite = invites.find(params[:id])
     authorize @invite, :destroy?
     @invite.expire!
     redirect_to invites_path
@@ -37,6 +37,10 @@ class InvitesController < ApplicationController
 
   private
 
+  def invites
+    Invite.where(user: current_user)
+  end
+
   def resource_params
     params.require(:invite).permit(:max_uses, :expires_in)
   end

app/controllers/media_proxy_controller.rb

@@ -8,6 +8,8 @@ class MediaProxyController < ApplicationController
       if lock.acquired?
         @media_attachment = MediaAttachment.remote.find(params[:id])
         redownload! if @media_attachment.needs_redownload? && !reject_media?
+      else
+        raise Mastodon::RaceConditionError
       end
     end
 

app/controllers/oauth/authorized_applications_controller.rb

@@ -8,6 +8,11 @@ class Oauth::AuthorizedApplicationsController < Doorkeeper::AuthorizedApplicatio
 
   include Localized
 
+  def destroy
+    Web::PushSubscription.unsubscribe_for(params[:id], current_resource_owner)
+    super
+  end
+
   private
 
   def store_current_location

app/controllers/oauth/tokens_controller.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class Oauth::TokensController < Doorkeeper::TokensController
+  def revoke
+    unsubscribe_for_token if authorized? && token.accessible?
+    super
+  end
+
+  private
+
+  def unsubscribe_for_token
+    Web::PushSubscription.where(access_token_id: token.id).delete_all
+  end
+end

app/controllers/remote_unfollows.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+class RemoteUnfollowsController < ApplicationController
+  layout 'modal'
+
+  before_action :authenticate_user!
+  before_action :set_body_classes
+
+  def create
+    @account = unfollow_attempt.try(:target_account)
+
+    if @account.nil?
+      render :error
+    else
+      render :success
+    end
+  rescue ActiveRecord::RecordNotFound, Mastodon::NotPermittedError
+    render :error
+  end
+
+  private
+
+  def unfollow_attempt
+    username, domain = acct_without_prefix.split('@')
+    UnfollowService.new.call(current_account, Account.find_remote!(username, domain))
+  end
+
+  def acct_without_prefix
+    acct_params.gsub(/\Aacct:/, '')
+  end
+
+  def acct_params
+    params.fetch(:acct, '')
+  end
+
+  def set_body_classes
+    @body_classes = 'modal-layout'
+  end
+end

app/controllers/settings/applications_controller.rb

@@ -8,7 +8,7 @@ class Settings::ApplicationsController < ApplicationController
   before_action :prepare_scopes, only: [:create, :update]
 
   def index
-    @applications = current_user.applications.page(params[:page])
+    @applications = current_user.applications.order(id: :desc).page(params[:page])
   end
 
   def new

app/controllers/settings/follower_domains_controller.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'sidekiq-bulk'
-
 class Settings::FollowerDomainsController < ApplicationController
   layout 'admin'
 
@@ -9,7 +7,7 @@ class Settings::FollowerDomainsController < ApplicationController
 
   def show
     @account = current_account
-    @domains = current_account.followers.reorder('MIN(follows.id) DESC').group('accounts.domain').select('accounts.domain, count(accounts.id) as accounts_from_domain').page(params[:page]).per(10)
+    @domains = current_account.followers.reorder(Arel.sql('MIN(follows.id) DESC')).group('accounts.domain').select('accounts.domain, count(accounts.id) as accounts_from_domain').page(params[:page]).per(10)
   end
 
   def update

app/controllers/settings/preferences_controller.rb

@@ -44,6 +44,7 @@ class Settings::PreferencesController < ApplicationController
       :setting_system_font_ui,
       :setting_noindex,
       :setting_theme,
+      :setting_hide_network,
       notification_emails: %i(follow follow_request reblog favourite mention digest),
       interactions: %i(must_be_follower must_be_following)
     )

app/controllers/settings/profiles_controller.rb

@@ -11,13 +11,16 @@ class Settings::ProfilesController < ApplicationController
   obfuscate_filename [:account, :avatar]
   obfuscate_filename [:account, :header]
 
-  def show; end
+  def show
+    @account.build_fields
+  end
 
   def update
     if UpdateAccountService.new.call(@account, account_params)
       ActivityPub::UpdateDistributionWorker.perform_async(@account.id)
       redirect_to settings_profile_path, notice: I18n.t('generic.changes_saved_msg')
     else
+      @account.build_fields
       render :show
     end
   end
@@ -25,7 +28,7 @@ class Settings::ProfilesController < ApplicationController
   private
 
   def account_params
-    params.require(:account).permit(:display_name, :note, :avatar, :header, :locked)
+    params.require(:account).permit(:display_name, :note, :avatar, :header, :locked, :bot, fields_attributes: [:name, :value])
   end
 
   def set_account

app/controllers/shares_controller.rb

@@ -15,6 +15,7 @@ class SharesController < ApplicationController
 
   def initial_state_params
     text = [params[:title], params[:text], params[:url]].compact.join(' ')
+
     {
       settings: Web::Setting.find_by(user: current_user)&.data || {},
       push_subscription: current_account.user.web_push_subscription(current_session),

app/controllers/statuses_controller.rb

@@ -4,6 +4,10 @@ class StatusesController < ApplicationController
   include SignatureAuthentication
   include Authorization
 
+  ANCESTORS_LIMIT         = 40
+  DESCENDANTS_LIMIT       = 60
+  DESCENDANTS_DEPTH_LIMIT = 20
+
   layout 'public'
 
   before_action :set_account
@@ -11,13 +15,14 @@ class StatusesController < ApplicationController
   before_action :set_link_headers
   before_action :check_account_suspension
   before_action :redirect_to_original, only: [:show]
+  before_action :set_referrer_policy_header, only: [:show]
   before_action :set_cache_headers
 
   def show
     respond_to do |format|
       format.html do
-        @ancestors   = @status.reply? ? cache_collection(@status.ancestors(current_account), Status) : []
-        @descendants = cache_collection(@status.descendants(current_account), Status)
+        set_ancestors
+        set_descendants
 
         render 'stream_entries/show'
       end
@@ -47,10 +52,77 @@ class StatusesController < ApplicationController
 
   private
 
+  def create_descendant_thread(depth, statuses)
+    if depth < DESCENDANTS_DEPTH_LIMIT
+      { statuses: statuses }
+    else
+      next_status = statuses.pop
+      { statuses: statuses, next_status: next_status }
+    end
+  end
+
   def set_account
     @account = Account.find_local!(params[:account_username])
   end
 
+  def set_ancestors
+    @ancestors     = @status.reply? ? cache_collection(@status.ancestors(ANCESTORS_LIMIT, current_account), Status) : []
+    @next_ancestor = @ancestors.size < ANCESTORS_LIMIT ? nil : @ancestors.shift
+  end
+
+  def set_descendants
+    @max_descendant_thread_id   = params[:max_descendant_thread_id]&.to_i
+    @since_descendant_thread_id = params[:since_descendant_thread_id]&.to_i
+
+    descendants = cache_collection(
+      @status.descendants(
+        DESCENDANTS_LIMIT,
+        current_account,
+        @max_descendant_thread_id,
+        @since_descendant_thread_id,
+        DESCENDANTS_DEPTH_LIMIT
+      ),
+      Status
+    )
+
+    @descendant_threads = []
+
+    if descendants.present?
+      statuses = [descendants.first]
+      depth    = 1
+
+      descendants.drop(1).each_with_index do |descendant, index|
+        if descendants[index].id == descendant.in_reply_to_id
+          depth += 1
+          statuses << descendant
+        else
+          @descendant_threads << create_descendant_thread(depth, statuses)
+
+          @descendant_threads.reverse_each do |descendant_thread|
+            statuses = descendant_thread[:statuses]
+
+            index = statuses.find_index do |thread_status|
+              thread_status.id == descendant.in_reply_to_id
+            end
+
+            if index.present?
+              depth += index - statuses.size
+              break
+            end
+
+            depth -= statuses.size
+          end
+
+          statuses = [descendant]
+        end
+      end
+
+      @descendant_threads << create_descendant_thread(depth, statuses)
+    end
+
+    @max_descendant_thread_id = @descendant_threads.pop[:statuses].first.id if descendants.size >= DESCENDANTS_LIMIT
+  end
+
   def set_link_headers
     response.headers['Link'] = LinkHeader.new(
       [
@@ -78,4 +150,9 @@ class StatusesController < ApplicationController
   def redirect_to_original
     redirect_to ::TagManager.instance.url_for(@status.reblog) if @status.reblog?
   end
+
+  def set_referrer_policy_header
+    return if @status.public_visibility? || @status.unlisted_visibility?
+    response.headers['Referrer-Policy'] = 'origin'
+  end
 end

app/controllers/stream_entries_controller.rb

@@ -15,8 +15,7 @@ class StreamEntriesController < ApplicationController
   def show
     respond_to do |format|
       format.html do
-        @ancestors   = @stream_entry.activity.reply? ? cache_collection(@stream_entry.activity.ancestors(current_account), Status) : []
-        @descendants = cache_collection(@stream_entry.activity.descendants(current_account), Status)
+        redirect_to short_account_status_url(params[:account_username], @stream_entry.activity) if @type == 'status'
       end
 
       format.atom do
@@ -24,6 +23,7 @@ class StreamEntriesController < ApplicationController
           skip_session!
           expires_in 3.minutes, public: true
         end
+
         render xml: OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.entry(@stream_entry, true))
       end
     end

app/controllers/tags_controller.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class TagsController < ApplicationController
+  PAGE_SIZE = 20
+
   before_action :set_body_classes
   before_action :set_instance_presenter
 
@@ -13,8 +15,15 @@ class TagsController < ApplicationController
         @initial_state_json   = serializable_resource.to_json
       end
 
+      format.rss do
+        @statuses = Status.as_tag_timeline(@tag).limit(PAGE_SIZE)
+        @statuses = cache_collection(@statuses, Status)
+
+        render xml: RSS::TagSerializer.render(@tag, @statuses)
+      end
+
       format.json do
-        @statuses = Status.as_tag_timeline(@tag, current_account, params[:local]).paginate_by_max_id(20, params[:max_id])
+        @statuses = Status.as_tag_timeline(@tag, current_account, params[:local]).paginate_by_max_id(PAGE_SIZE, params[:max_id])
         @statuses = cache_collection(@statuses, Status)
 
         render json: collection_presenter,

app/helpers/admin/account_moderation_notes_helper.rb

@@ -1,4 +1,26 @@
 # frozen_string_literal: true
 
 module Admin::AccountModerationNotesHelper
+  def admin_account_link_to(account)
+    link_to admin_account_path(account.id), class: name_tag_classes(account) do
+      safe_join([
+                  image_tag(account.avatar.url, width: 15, height: 15, alt: display_name(account), class: 'avatar'),
+                  content_tag(:span, account.acct, class: 'username'),
+                ], ' ')
+    end
+  end
+
+  def admin_account_inline_link_to(account)
+    link_to admin_account_path(account.id), class: name_tag_classes(account, true) do
+      content_tag(:span, account.acct, class: 'username')
+    end
+  end
+
+  private
+
+  def name_tag_classes(account, inline = false)
+    classes = [inline ? 'inline-name-tag' : 'name-tag']
+    classes << 'suspended' if account.suspended?
+    classes.join(' ')
+  end
 end

app/helpers/admin/action_logs_helper.rb

@@ -45,6 +45,8 @@ module Admin::ActionLogsHelper
       log.recorded_changes.slice('domain', 'visible_in_picker')
     elsif log.target_type == 'User' && [:promote, :demote].include?(log.action)
       log.recorded_changes.slice('moderator', 'admin')
+    elsif log.target_type == 'User' && [:change_email].include?(log.action)
+      log.recorded_changes.slice('email', 'unconfirmed_email')
     elsif log.target_type == 'DomainBlock'
       log.recorded_changes.slice('severity', 'reject_media')
     elsif log.target_type == 'Status' && log.action == :update
@@ -84,9 +86,9 @@ module Admin::ActionLogsHelper
       'positive'
     when :create
       opposite_verbs?(log) ? 'negative' : 'positive'
-    when :update, :reset_password, :disable_2fa, :memorialize
+    when :update, :reset_password, :disable_2fa, :memorialize, :change_email
       'neutral'
-    when :demote, :silence, :disable, :suspend
+    when :demote, :silence, :disable, :suspend, :remove_avatar, :reopen
       'negative'
     when :destroy
       opposite_verbs?(log) ? 'positive' : 'negative'

app/helpers/application_helper.rb

@@ -63,4 +63,8 @@ module ApplicationHelper
   def opengraph(property, content)
     tag(:meta, content: content, property: property)
   end
+
+  def react_component(name, props = {})
+    content_tag(:div, nil, data: { component: name.to_s.camelcase, props: Oj.dump(props) })
+  end
 end

app/helpers/jsonld_helper.rb

@@ -5,6 +5,10 @@ module JsonLdHelper
     haystack.is_a?(Array) ? haystack.include?(needle) : haystack == needle
   end
 
+  def equals_or_includes_any?(haystack, needles)
+    needles.any? { |needle| equals_or_includes?(haystack, needle) }
+  end
+
   def first_of_value(value)
     value.is_a?(Array) ? value.first : value
   end
@@ -44,25 +48,29 @@ module JsonLdHelper
   end
 
   def canonicalize(json)
-    graph = RDF::Graph.new << JSON::LD::API.toRdf(json)
+    graph = RDF::Graph.new << JSON::LD::API.toRdf(json, documentLoader: method(:load_jsonld_context))
     graph.dump(:normalize)
   end
 
-  def fetch_resource(uri, id)
+  def fetch_resource(uri, id, on_behalf_of = nil)
     unless id
-      json = fetch_resource_without_id_validation(uri)
+      json = fetch_resource_without_id_validation(uri, on_behalf_of)
       return unless json
       uri = json['id']
     end
 
-    json = fetch_resource_without_id_validation(uri)
+    json = fetch_resource_without_id_validation(uri, on_behalf_of)
     json.present? && json['id'] == uri ? json : nil
   end
 
-  def fetch_resource_without_id_validation(uri)
-    response = build_request(uri).perform
-    return if response.code != 200
-    body_to_json(response.to_s)
+  def fetch_resource_without_id_validation(uri, on_behalf_of = nil)
+    build_request(uri, on_behalf_of).perform do |response|
+      return body_to_json(response.body_with_limit) if response.code == 200
+    end
+    # If request failed, retry without doing it on behalf of a user
+    build_request(uri).perform do |response|
+      response.code == 200 ? body_to_json(response.body_with_limit) : nil
+    end
   end
 
   def body_to_json(body)
@@ -81,9 +89,25 @@ module JsonLdHelper
 
   private
 
-  def build_request(uri)
+  def build_request(uri, on_behalf_of = nil)
     request = Request.new(:get, uri)
+    request.on_behalf_of(on_behalf_of) if on_behalf_of
     request.add_headers('Accept' => 'application/activity+json, application/ld+json')
     request
   end
+
+  def load_jsonld_context(url, _options = {}, &_block)
+    json = Rails.cache.fetch("jsonld:context:#{url}", expires_in: 30.days, raw: true) do
+      request = Request.new(:get, url)
+      request.add_headers('Accept' => 'application/ld+json')
+
+      request.perform do |res|
+        raise JSON::LD::JsonLdError::LoadingDocumentFailed unless res.code == 200 && res.mime_type == 'application/ld+json'
+        res.body_with_limit
+      end
+    end
+
+    doc = JSON::LD::API::RemoteDocument.new(url, json)
+    block_given? ? yield(doc) : doc
+  end
 end

app/helpers/settings_helper.rb

@@ -6,13 +6,16 @@ module SettingsHelper
     ar: 'العربية',
     bg: 'Български',
     ca: 'Català',
+    co: 'Corsu',
     de: 'Deutsch',
+    el: 'Ελληνικά',
     eo: 'Esperanto',
     es: 'Español',
+    eu: 'Euskara',
     fa: 'فارسی',
-    gl: 'Galego',
     fi: 'Suomi',
     fr: 'Français',
+    gl: 'Galego',
     he: 'עברית',
     hr: 'Hrvatski',
     hu: 'Magyar',
@@ -30,9 +33,11 @@ module SettingsHelper
     'pt-BR': 'Português do Brasil',
     ru: 'Русский',
     sk: 'Slovensky',
+    sl: 'Slovenščina',
     sr: 'Српски',
     'sr-Latn': 'Srpski (latinica)',
     sv: 'Svenska',
+    te: 'తెలుగు',
     th: 'ภาษาไทย',
     tr: 'Türkçe',
     uk: 'Українська',

app/helpers/stream_entries_helper.rb

@@ -4,25 +4,29 @@ module StreamEntriesHelper
   EMBEDDED_CONTROLLER = 'statuses'
   EMBEDDED_ACTION = 'embed'
 
-  def display_name(account)
-    account.display_name.presence || account.username
+  def display_name(account, **options)
+    if options[:custom_emojify]
+      Formatter.instance.format_display_name(account, options)
+    else
+      account.display_name.presence || account.username
+    end
   end
 
   def account_description(account)
     prepend_str = [
       [
         number_to_human(account.statuses_count, strip_insignificant_zeros: true),
-        t('accounts.posts'),
+        I18n.t('accounts.posts'),
       ].join(' '),
 
       [
         number_to_human(account.following_count, strip_insignificant_zeros: true),
-        t('accounts.following'),
+        I18n.t('accounts.following'),
       ].join(' '),
 
       [
         number_to_human(account.followers_count, strip_insignificant_zeros: true),
-        t('accounts.followers'),
+        I18n.t('accounts.followers'),
       ].join(' '),
     ].join(', ')
 
@@ -40,16 +44,16 @@ module StreamEntriesHelper
       end
     end
 
-    text = attachments.to_a.reject { |_, value| value.zero? }.map { |key, value| t("statuses.attached.#{key}", count: value) }.join(' · ')
+    text = attachments.to_a.reject { |_, value| value.zero? }.map { |key, value| I18n.t("statuses.attached.#{key}", count: value) }.join(' · ')
 
     return if text.blank?
 
-    t('statuses.attached.description', attached: text)
+    I18n.t('statuses.attached.description', attached: text)
   end
 
   def status_text_summary(status)
     return if status.spoiler_text.blank?
-    t('statuses.content_warning', warning: status.spoiler_text)
+    I18n.t('statuses.content_warning', warning: status.spoiler_text)
   end
 
   def status_description(status)
@@ -113,6 +117,19 @@ module StreamEntriesHelper
     end
   end
 
+  def fa_visibility_icon(status)
+    case status.visibility
+    when 'public'
+      fa_icon 'globe fw'
+    when 'unlisted'
+      fa_icon 'unlock-alt fw'
+    when 'private'
+      fa_icon 'lock fw'
+    when 'direct'
+      fa_icon 'envelope fw'
+    end
+  end
+
   private
 
   def simplified_text(text)

app/javascript/mastodon/actions/accounts.js

@@ -1,4 +1,6 @@
 import api, { getLinks } from '../api';
+import openDB from '../storage/db';
+import { importAccount, importFetchedAccount, importFetchedAccounts } from './importer';
 
 export const ACCOUNT_FETCH_REQUEST = 'ACCOUNT_FETCH_REQUEST';
 export const ACCOUNT_FETCH_SUCCESS = 'ACCOUNT_FETCH_SUCCESS';
@@ -64,6 +66,24 @@ export const FOLLOW_REQUEST_REJECT_REQUEST = 'FOLLOW_REQUEST_REJECT_REQUEST';
 export const FOLLOW_REQUEST_REJECT_SUCCESS = 'FOLLOW_REQUEST_REJECT_SUCCESS';
 export const FOLLOW_REQUEST_REJECT_FAIL    = 'FOLLOW_REQUEST_REJECT_FAIL';
 
+function getFromDB(dispatch, getState, index, id) {
+  return new Promise((resolve, reject) => {
+    const request = index.get(id);
+
+    request.onerror = reject;
+
+    request.onsuccess = () => {
+      if (!request.result) {
+        reject();
+        return;
+      }
+
+      dispatch(importAccount(request.result));
+      resolve(request.result.moved && getFromDB(dispatch, getState, index, request.result.moved));
+    };
+  });
+}
+
 export function fetchAccount(id) {
   return (dispatch, getState) => {
     dispatch(fetchRelationships([id]));
@@ -74,8 +94,18 @@ export function fetchAccount(id) {
 
     dispatch(fetchAccountRequest(id));
 
-    api(getState).get(`/api/v1/accounts/${id}`).then(response => {
-      dispatch(fetchAccountSuccess(response.data));
+    openDB().then(db => getFromDB(
+      dispatch,
+      getState,
+      db.transaction('accounts', 'read').objectStore('accounts').index('id'),
+      id
+    ).then(() => db.close(), error => {
+      db.close();
+      throw error;
+    })).catch(() => api(getState).get(`/api/v1/accounts/${id}`).then(response => {
+      dispatch(importFetchedAccount(response.data));
+    })).then(() => {
+      dispatch(fetchAccountSuccess());
     }).catch(error => {
       dispatch(fetchAccountFail(id, error));
     });
@@ -89,10 +119,9 @@ export function fetchAccountRequest(id) {
   };
 };
 
-export function fetchAccountSuccess(account) {
+export function fetchAccountSuccess() {
   return {
     type: ACCOUNT_FETCH_SUCCESS,
-    account,
   };
 };
 
@@ -319,6 +348,7 @@ export function fetchFollowers(id) {
     api(getState).get(`/api/v1/accounts/${id}/followers`).then(response => {
       const next = getLinks(response).refs.find(link => link.rel === 'next');
 
+      dispatch(importFetchedAccounts(response.data));
       dispatch(fetchFollowersSuccess(id, response.data, next ? next.uri : null));
       dispatch(fetchRelationships(response.data.map(item => item.id)));
     }).catch(error => {
@@ -364,6 +394,7 @@ export function expandFollowers(id) {
     api(getState).get(url).then(response => {
       const next = getLinks(response).refs.find(link => link.rel === 'next');
 
+      dispatch(importFetchedAccounts(response.data));
       dispatch(expandFollowersSuccess(id, response.data, next ? next.uri : null));
       dispatch(fetchRelationships(response.data.map(item => item.id)));
     }).catch(error => {
@@ -403,6 +434,7 @@ export function fetchFollowing(id) {
     api(getState).get(`/api/v1/accounts/${id}/following`).then(response => {
       const next = getLinks(response).refs.find(link => link.rel === 'next');
 
+      dispatch(importFetchedAccounts(response.data));
       dispatch(fetchFollowingSuccess(id, response.data, next ? next.uri : null));
       dispatch(fetchRelationships(response.data.map(item => item.id)));
     }).catch(error => {
@@ -448,6 +480,7 @@ export function expandFollowing(id) {
     api(getState).get(url).then(response => {
       const next = getLinks(response).refs.find(link => link.rel === 'next');
 
+      dispatch(importFetchedAccounts(response.data));
       dispatch(expandFollowingSuccess(id, response.data, next ? next.uri : null));
       dispatch(fetchRelationships(response.data.map(item => item.id)));
     }).catch(error => {
@@ -529,6 +562,7 @@ export function fetchFollowRequests() {
 
     api(getState).get('/api/v1/follow_requests').then(response => {
       const next = getLinks(response).refs.find(link => link.rel === 'next');
+      dispatch(importFetchedAccounts(response.data));
       dispatch(fetchFollowRequestsSuccess(response.data, next ? next.uri : null));
     }).catch(error => dispatch(fetchFollowRequestsFail(error)));
   };
@@ -567,6 +601,7 @@ export function expandFollowRequests() {
 
     api(getState).get(url).then(response => {
       const next = getLinks(response).refs.find(link => link.rel === 'next');
+      dispatch(importFetchedAccounts(response.data));
       dispatch(expandFollowRequestsSuccess(response.data, next ? next.uri : null));
     }).catch(error => dispatch(expandFollowRequestsFail(error)));
   };

app/javascript/mastodon/actions/alerts.js

@@ -1,3 +1,10 @@
+import { defineMessages } from 'react-intl';
+
+const messages = defineMessages({
+  unexpectedTitle: { id: 'alert.unexpected.title', defaultMessage: 'Oops!' },
+  unexpectedMessage: { id: 'alert.unexpected.message', defaultMessage: 'An unexpected error occurred.' },
+});
+
 export const ALERT_SHOW    = 'ALERT_SHOW';
 export const ALERT_DISMISS = 'ALERT_DISMISS';
 export const ALERT_CLEAR   = 'ALERT_CLEAR';
@@ -22,3 +29,21 @@ export function showAlert(title, message) {
     message,
   };
 };
+
+export function showAlertForError(error) {
+  if (error.response) {
+    const { data, status, statusText } = error.response;
+
+    let message = statusText;
+    let title   = `${status}`;
+
+    if (data.error) {
+      message = data.error;
+    }
+
+    return showAlert(title, message);
+  } else {
+    console.error(error);
+    return showAlert(messages.unexpectedTitle, messages.unexpectedMessage);
+  }
+}

app/javascript/mastodon/actions/blocks.js

@@ -1,5 +1,6 @@
 import api, { getLinks } from '../api';
 import { fetchRelationships } from './accounts';
+import { importFetchedAccounts } from './importer';
 
 export const BLOCKS_FETCH_REQUEST = 'BLOCKS_FETCH_REQUEST';
 export const BLOCKS_FETCH_SUCCESS = 'BLOCKS_FETCH_SUCCESS';
@@ -15,6 +16,7 @@ export function fetchBlocks() {
 
     api(getState).get('/api/v1/blocks').then(response => {
       const next = getLinks(response).refs.find(link => link.rel === 'next');
+      dispatch(importFetchedAccounts(response.data));
       dispatch(fetchBlocksSuccess(response.data, next ? next.uri : null));
       dispatch(fetchRelationships(response.data.map(item => item.id)));
     }).catch(error => dispatch(fetchBlocksFail(error)));
@@ -54,6 +56,7 @@ export function expandBlocks() {
 
     api(getState).get(url).then(response => {
       const next = getLinks(response).refs.find(link => link.rel === 'next');
+      dispatch(importFetchedAccounts(response.data));
       dispatch(expandBlocksSuccess(response.data, next ? next.uri : null));
       dispatch(fetchRelationships(response.data.map(item => item.id)));
     }).catch(error => dispatch(expandBlocksFail(error)));

app/javascript/mastodon/actions/columns.js

@@ -1,8 +1,9 @@
 import { saveSettings } from './settings';
 
-export const COLUMN_ADD    = 'COLUMN_ADD';
-export const COLUMN_REMOVE = 'COLUMN_REMOVE';
-export const COLUMN_MOVE   = 'COLUMN_MOVE';
+export const COLUMN_ADD           = 'COLUMN_ADD';
+export const COLUMN_REMOVE        = 'COLUMN_REMOVE';
+export const COLUMN_MOVE          = 'COLUMN_MOVE';
+export const COLUMN_PARAMS_CHANGE = 'COLUMN_PARAMS_CHANGE';
 
 export function addColumn(id, params) {
   return dispatch => {
@@ -38,3 +39,15 @@ export function moveColumn(uuid, direction) {
     dispatch(saveSettings());
   };
 };
+
+export function changeColumnParams(uuid, params) {
+  return dispatch => {
+    dispatch({
+      type: COLUMN_PARAMS_CHANGE,
+      uuid,
+      params,
+    });
+
+    dispatch(saveSettings());
+  };
+}

app/javascript/mastodon/actions/compose.js

@@ -1,15 +1,15 @@
 import api from '../api';
+import { CancelToken, isCancel } from 'axios';
 import { throttle } from 'lodash';
 import { search as emojiSearch } from '../features/emoji/emoji_mart_search_light';
 import { tagHistory } from '../settings';
 import { useEmoji } from './emojis';
+import resizeImage from '../utils/resize_image';
+import { importFetchedAccounts } from './importer';
+import { updateTimeline } from './timelines';
+import { showAlertForError } from './alerts';
 
-import {
-  updateTimeline,
-  refreshHomeTimeline,
-  refreshCommunityTimeline,
-  refreshPublicTimeline,
-} from './timelines';
+let cancelFetchComposeSuggestionsAccounts;
 
 export const COMPOSE_CHANGE          = 'COMPOSE_CHANGE';
 export const COMPOSE_SUBMIT_REQUEST  = 'COMPOSE_SUBMIT_REQUEST';
@@ -17,6 +17,7 @@ export const COMPOSE_SUBMIT_SUCCESS  = 'COMPOSE_SUBMIT_SUCCESS';
 export const COMPOSE_SUBMIT_FAIL     = 'COMPOSE_SUBMIT_FAIL';
 export const COMPOSE_REPLY           = 'COMPOSE_REPLY';
 export const COMPOSE_REPLY_CANCEL    = 'COMPOSE_REPLY_CANCEL';
+export const COMPOSE_DIRECT          = 'COMPOSE_DIRECT';
 export const COMPOSE_MENTION         = 'COMPOSE_MENTION';
 export const COMPOSE_RESET           = 'COMPOSE_RESET';
 export const COMPOSE_UPLOAD_REQUEST  = 'COMPOSE_UPLOAD_REQUEST';
@@ -93,6 +94,19 @@ export function mentionCompose(account, router) {
   };
 };
 
+export function directCompose(account, router) {
+  return (dispatch, getState) => {
+    dispatch({
+      type: COMPOSE_DIRECT,
+      account: account,
+    });
+
+    if (!getState().getIn(['compose', 'mounted'])) {
+      router.push('/statuses/new');
+    }
+  };
+};
+
 export function submitCompose() {
   return function (dispatch, getState) {
     const status = getState().getIn(['compose', 'text'], '');
@@ -121,19 +135,19 @@ export function submitCompose() {
 
       // To make the app more responsive, immediately get the status into the columns
 
-      const insertOrRefresh = (timelineId, refreshAction) => {
-        if (getState().getIn(['timelines', timelineId, 'online'])) {
+      const insertIfOnline = (timelineId) => {
+        if (getState().getIn(['timelines', timelineId, 'items', 0]) !== null) {
           dispatch(updateTimeline(timelineId, { ...response.data }));
-        } else if (getState().getIn(['timelines', timelineId, 'loaded'])) {
-          dispatch(refreshAction());
         }
       };
 
-      insertOrRefresh('home', refreshHomeTimeline);
+      insertIfOnline('home');
 
       if (response.data.in_reply_to_id === null && response.data.visibility === 'public') {
-        insertOrRefresh('community', refreshCommunityTimeline);
-        insertOrRefresh('public', refreshPublicTimeline);
+        insertIfOnline('community');
+        insertIfOnline('public');
+      } else if (response.data.visibility === 'direct') {
+        insertIfOnline('direct');
       }
     }).catch(function (error) {
       dispatch(submitComposeFail(error));
@@ -169,18 +183,14 @@ export function uploadCompose(files) {
 
     dispatch(uploadComposeRequest());
 
-    let data = new FormData();
-    data.append('file', files[0]);
+    resizeImage(files[0]).then(file => {
+      const data = new FormData();
+      data.append('file', file);
 
-    api(getState).post('/api/v1/media', data, {
-      onUploadProgress: function (e) {
-        dispatch(uploadComposeProgress(e.loaded, e.total));
-      },
-    }).then(function (response) {
-      dispatch(uploadComposeSuccess(response.data));
-    }).catch(function (error) {
-      dispatch(uploadComposeFail(error));
-    });
+      return api(getState).post('/api/v1/media', data, {
+        onUploadProgress: ({ loaded, total }) => dispatch(uploadComposeProgress(loaded, total)),
+      }).then(({ data }) => dispatch(uploadComposeSuccess(data)));
+    }).catch(error => dispatch(uploadComposeFail(error)));
   };
 };
 
@@ -257,20 +267,34 @@ export function undoUploadCompose(media_id) {
 };
 
 export function clearComposeSuggestions() {
+  if (cancelFetchComposeSuggestionsAccounts) {
+    cancelFetchComposeSuggestionsAccounts();
+  }
   return {
     type: COMPOSE_SUGGESTIONS_CLEAR,
   };
 };
 
 const fetchComposeSuggestionsAccounts = throttle((dispatch, getState, token) => {
+  if (cancelFetchComposeSuggestionsAccounts) {
+    cancelFetchComposeSuggestionsAccounts();
+  }
   api(getState).get('/api/v1/accounts/search', {
+    cancelToken: new CancelToken(cancel => {
+      cancelFetchComposeSuggestionsAccounts = cancel;
+    }),
     params: {
       q: token.slice(1),
       resolve: false,
       limit: 4,
     },
   }).then(response => {
+    dispatch(importFetchedAccounts(response.data));
     dispatch(readyComposeSuggestionsAccounts(token, response.data));
+  }).catch(error => {
+    if (!isCancel(error)) {
+      dispatch(showAlertForError(error));
+    }
   });
 }, 200, { leading: true, trailing: true });
 
@@ -421,11 +445,12 @@ export function changeComposeVisibility(value) {
   };
 };
 
-export function insertEmojiCompose(position, emoji) {
+export function insertEmojiCompose(position, emoji, needsSpace) {
   return {
     type: COMPOSE_EMOJI_INSERT,
     position,
     emoji,
+    needsSpace,
   };
 };
 

app/javascript/mastodon/actions/custom_emojis.js

@@ -0,0 +1,37 @@
+import api from '../api';
+
+export const CUSTOM_EMOJIS_FETCH_REQUEST = 'CUSTOM_EMOJIS_FETCH_REQUEST';
+export const CUSTOM_EMOJIS_FETCH_SUCCESS = 'CUSTOM_EMOJIS_FETCH_SUCCESS';
+export const CUSTOM_EMOJIS_FETCH_FAIL = 'CUSTOM_EMOJIS_FETCH_FAIL';
+
+export function fetchCustomEmojis() {
+  return (dispatch, getState) => {
+    dispatch(fetchCustomEmojisRequest());
+
+    api(getState).get('/api/v1/custom_emojis').then(response => {
+      dispatch(fetchCustomEmojisSuccess(response.data));
+    }).catch(error => {
+      dispatch(fetchCustomEmojisFail(error));
+    });
+  };
+};
+
+export function fetchCustomEmojisRequest() {
+  return {
+    type: CUSTOM_EMOJIS_FETCH_REQUEST,
+  };
+};
+
+export function fetchCustomEmojisSuccess(custom_emojis) {
+  return {
+    type: CUSTOM_EMOJIS_FETCH_SUCCESS,
+    custom_emojis,
+  };
+};
+
+export function fetchCustomEmojisFail(error) {
+  return {
+    type: CUSTOM_EMOJIS_FETCH_FAIL,
+    error,
+  };
+};

app/javascript/mastodon/actions/domain_blocks.js

@@ -12,12 +12,18 @@ export const DOMAIN_BLOCKS_FETCH_REQUEST = 'DOMAIN_BLOCKS_FETCH_REQUEST';
 export const DOMAIN_BLOCKS_FETCH_SUCCESS = 'DOMAIN_BLOCKS_FETCH_SUCCESS';
 export const DOMAIN_BLOCKS_FETCH_FAIL    = 'DOMAIN_BLOCKS_FETCH_FAIL';
 
-export function blockDomain(domain, accountId) {
+export const DOMAIN_BLOCKS_EXPAND_REQUEST = 'DOMAIN_BLOCKS_EXPAND_REQUEST';
+export const DOMAIN_BLOCKS_EXPAND_SUCCESS = 'DOMAIN_BLOCKS_EXPAND_SUCCESS';
+export const DOMAIN_BLOCKS_EXPAND_FAIL    = 'DOMAIN_BLOCKS_EXPAND_FAIL';
+
+export function blockDomain(domain) {
   return (dispatch, getState) => {
     dispatch(blockDomainRequest(domain));
 
     api(getState).post('/api/v1/domain_blocks', { domain }).then(() => {
-      dispatch(blockDomainSuccess(domain, accountId));
+      const at_domain = '@' + domain;
+      const accounts = getState().get('accounts').filter(item => item.get('acct').endsWith(at_domain)).valueSeq().map(item => item.get('id'));
+      dispatch(blockDomainSuccess(domain, accounts));
     }).catch(err => {
       dispatch(blockDomainFail(domain, err));
     });
@@ -31,11 +37,11 @@ export function blockDomainRequest(domain) {
   };
 };
 
-export function blockDomainSuccess(domain, accountId) {
+export function blockDomainSuccess(domain, accounts) {
   return {
     type: DOMAIN_BLOCK_SUCCESS,
     domain,
-    accountId,
+    accounts,
   };
 };
 
@@ -47,12 +53,14 @@ export function blockDomainFail(domain, error) {
   };
 };
 
-export function unblockDomain(domain, accountId) {
+export function unblockDomain(domain) {
   return (dispatch, getState) => {
     dispatch(unblockDomainRequest(domain));
 
     api(getState).delete('/api/v1/domain_blocks', { params: { domain } }).then(() => {
-      dispatch(unblockDomainSuccess(domain, accountId));
+      const at_domain = '@' + domain;
+      const accounts = getState().get('accounts').filter(item => item.get('acct').endsWith(at_domain)).valueSeq().map(item => item.get('id'));
+      dispatch(unblockDomainSuccess(domain, accounts));
     }).catch(err => {
       dispatch(unblockDomainFail(domain, err));
     });
@@ -66,11 +74,11 @@ export function unblockDomainRequest(domain) {
   };
 };
 
-export function unblockDomainSuccess(domain, accountId) {
+export function unblockDomainSuccess(domain, accounts) {
   return {
     type: DOMAIN_UNBLOCK_SUCCESS,
     domain,
-    accountId,
+    accounts,
   };
 };
 
@@ -86,7 +94,7 @@ export function fetchDomainBlocks() {
   return (dispatch, getState) => {
     dispatch(fetchDomainBlocksRequest());
 
-    api(getState).get().then(response => {
+    api(getState).get('/api/v1/domain_blocks').then(response => {
       const next = getLinks(response).refs.find(link => link.rel === 'next');
       dispatch(fetchDomainBlocksSuccess(response.data, next ? next.uri : null));
     }).catch(err => {
@@ -115,3 +123,43 @@ export function fetchDomainBlocksFail(error) {
     error,
   };
 };
+
+export function expandDomainBlocks() {
+  return (dispatch, getState) => {
+    const url = getState().getIn(['domain_lists', 'blocks', 'next']);
+
+    if (url === null) {
+      return;
+    }
+
+    dispatch(expandDomainBlocksRequest());
+
+    api(getState).get(url).then(response => {
+      const next = getLinks(response).refs.find(link => link.rel === 'next');
+      dispatch(expandDomainBlocksSuccess(response.data, next ? next.uri : null));
+    }).catch(err => {
+      dispatch(expandDomainBlocksFail(err));
+    });
+  };
+};
+
+export function expandDomainBlocksRequest() {
+  return {
+    type: DOMAIN_BLOCKS_EXPAND_REQUEST,
+  };
+};
+
+export function expandDomainBlocksSuccess(domains, next) {
+  return {
+    type: DOMAIN_BLOCKS_EXPAND_SUCCESS,
+    domains,
+    next,
+  };
+};
+
+export function expandDomainBlocksFail(error) {
+  return {
+    type: DOMAIN_BLOCKS_EXPAND_FAIL,
+    error,
+  };
+};

app/javascript/mastodon/actions/favourites.js

@@ -1,4 +1,5 @@
 import api, { getLinks } from '../api';
+import { importFetchedStatuses } from './importer';
 
 export const FAVOURITED_STATUSES_FETCH_REQUEST = 'FAVOURITED_STATUSES_FETCH_REQUEST';
 export const FAVOURITED_STATUSES_FETCH_SUCCESS = 'FAVOURITED_STATUSES_FETCH_SUCCESS';
@@ -18,6 +19,7 @@ export function fetchFavouritedStatuses() {
 
     api(getState).get('/api/v1/favourites').then(response => {
       const next = getLinks(response).refs.find(link => link.rel === 'next');
+      dispatch(importFetchedStatuses(response.data));
       dispatch(fetchFavouritedStatusesSuccess(response.data, next ? next.uri : null));
     }).catch(error => {
       dispatch(fetchFavouritedStatusesFail(error));
@@ -58,6 +60,7 @@ export function expandFavouritedStatuses() {
 
     api(getState).get(url).then(response => {
       const next = getLinks(response).refs.find(link => link.rel === 'next');
+      dispatch(importFetchedStatuses(response.data));
       dispatch(expandFavouritedStatusesSuccess(response.data, next ? next.uri : null));
     }).catch(error => {
       dispatch(expandFavouritedStatusesFail(error));

app/javascript/mastodon/actions/importer/index.js

@@ -0,0 +1,77 @@
+import { autoPlayGif } from '../../initial_state';
+import { putAccounts, putStatuses } from '../../storage/modifier';
+import { normalizeAccount, normalizeStatus } from './normalizer';
+
+export const ACCOUNT_IMPORT = 'ACCOUNT_IMPORT';
+export const ACCOUNTS_IMPORT = 'ACCOUNTS_IMPORT';
+export const STATUS_IMPORT = 'STATUS_IMPORT';
+export const STATUSES_IMPORT = 'STATUSES_IMPORT';
+
+function pushUnique(array, object) {
+  if (array.every(element => element.id !== object.id)) {
+    array.push(object);
+  }
+}
+
+export function importAccount(account) {
+  return { type: ACCOUNT_IMPORT, account };
+}
+
+export function importAccounts(accounts) {
+  return { type: ACCOUNTS_IMPORT, accounts };
+}
+
+export function importStatus(status) {
+  return { type: STATUS_IMPORT, status };
+}
+
+export function importStatuses(statuses) {
+  return { type: STATUSES_IMPORT, statuses };
+}
+
+export function importFetchedAccount(account) {
+  return importFetchedAccounts([account]);
+}
+
+export function importFetchedAccounts(accounts) {
+  const normalAccounts = [];
+
+  function processAccount(account) {
+    pushUnique(normalAccounts, normalizeAccount(account));
+
+    if (account.moved) {
+      processAccount(account.moved);
+    }
+  }
+
+  accounts.forEach(processAccount);
+  putAccounts(normalAccounts, !autoPlayGif);
+
+  return importAccounts(normalAccounts);
+}
+
+export function importFetchedStatus(status) {
+  return importFetchedStatuses([status]);
+}
+
+export function importFetchedStatuses(statuses) {
+  return (dispatch, getState) => {
+    const accounts = [];
+    const normalStatuses = [];
+
+    function processStatus(status) {
+      pushUnique(normalStatuses, normalizeStatus(status, getState().getIn(['statuses', status.id])));
+      pushUnique(accounts, status.account);
+
+      if (status.reblog && status.reblog.id) {
+        processStatus(status.reblog);
+      }
+    }
+
+    statuses.forEach(processStatus);
+    putStatuses(normalStatuses);
+
+    dispatch(importFetchedAccounts(accounts));
+    dispatch(importStatuses(normalStatuses));
+  };
+}

app/javascript/mastodon/actions/importer/normalizer.js

@@ -0,0 +1,63 @@
+import escapeTextContentForBrowser from 'escape-html';
+import emojify from '../../features/emoji/emoji';
+import { unescapeHTML } from '../../utils/html';
+
+const domParser = new DOMParser();
+
+const makeEmojiMap = record => record.emojis.reduce((obj, emoji) => {
+  obj[`:${emoji.shortcode}:`] = emoji;
+  return obj;
+}, {});
+
+export function normalizeAccount(account) {
+  account = { ...account };
+
+  const emojiMap = makeEmojiMap(account);
+  const displayName = account.display_name.length === 0 ? account.username : account.display_name;
+
+  account.display_name_html = emojify(escapeTextContentForBrowser(displayName), emojiMap);
+  account.note_emojified = emojify(account.note, emojiMap);
+
+  if (account.fields) {
+    account.fields = account.fields.map(pair => ({
+      ...pair,
+      name_emojified: emojify(escapeTextContentForBrowser(pair.name)),
+      value_emojified: emojify(pair.value, emojiMap),
+      value_plain: unescapeHTML(pair.value),
+    }));
+  }
+
+  if (account.moved) {
+    account.moved = account.moved.id;
+  }
+
+  return account;
+}
+
+export function normalizeStatus(status, normalOldStatus) {
+  const normalStatus   = { ...status };
+  normalStatus.account = status.account.id;
+
+  if (status.reblog && status.reblog.id) {
+    normalStatus.reblog = status.reblog.id;
+  }
+
+  // Only calculate these values when status first encountered
+  // Otherwise keep the ones already in the reducer
+  if (normalOldStatus) {
+    normalStatus.search_index = normalOldStatus.get('search_index');
+    normalStatus.contentHtml = normalOldStatus.get('contentHtml');
+    normalStatus.spoilerHtml = normalOldStatus.get('spoilerHtml');
+    normalStatus.hidden = normalOldStatus.get('hidden');
+  } else {
+    const searchContent = [status.spoiler_text, status.content].join('\n\n').replace(/<br\s*\/?>/g, '\n').replace(/<\/p><p>/g, '\n\n');
+    const emojiMap = makeEmojiMap(normalStatus);
+
+    normalStatus.search_index = domParser.parseFromString(searchContent, 'text/html').documentElement.textContent;
+    normalStatus.contentHtml  = emojify(normalStatus.content, emojiMap);
+    normalStatus.spoilerHtml  = emojify(escapeTextContentForBrowser(normalStatus.spoiler_text || ''), emojiMap);
+    normalStatus.hidden       = normalStatus.sensitive;
+  }
+
+  return normalStatus;
+}

app/javascript/mastodon/actions/interactions.js

@@ -1,4 +1,5 @@
 import api from '../api';
+import { importFetchedAccounts, importFetchedStatus } from './importer';
 
 export const REBLOG_REQUEST = 'REBLOG_REQUEST';
 export const REBLOG_SUCCESS = 'REBLOG_SUCCESS';
@@ -39,7 +40,8 @@ export function reblog(status) {
     api(getState).post(`/api/v1/statuses/${status.get('id')}/reblog`).then(function (response) {
       // The reblog API method returns a new status wrapped around the original. In this case we are only
       // interested in how the original is modified, hence passing it skipping the wrapper
-      dispatch(reblogSuccess(status, response.data.reblog));
+      dispatch(importFetchedStatus(response.data.reblog));
+      dispatch(reblogSuccess(status));
     }).catch(function (error) {
       dispatch(reblogFail(status, error));
     });
@@ -51,7 +53,8 @@ export function unreblog(status) {
     dispatch(unreblogRequest(status));
 
     api(getState).post(`/api/v1/statuses/${status.get('id')}/unreblog`).then(response => {
-      dispatch(unreblogSuccess(status, response.data));
+      dispatch(importFetchedStatus(response.data));
+      dispatch(unreblogSuccess(status));
     }).catch(error => {
       dispatch(unreblogFail(status, error));
     });
@@ -66,11 +69,10 @@ export function reblogRequest(status) {
   };
 };
 
-export function reblogSuccess(status, response) {
+export function reblogSuccess(status) {
   return {
     type: REBLOG_SUCCESS,
     status: status,
-    response: response,
     skipLoading: true,
   };
 };
@@ -92,11 +94,10 @@ export function unreblogRequest(status) {
   };
 };
 
-export function unreblogSuccess(status, response) {
+export function unreblogSuccess(status) {
   return {
     type: UNREBLOG_SUCCESS,
     status: status,
-    response: response,
     skipLoading: true,
   };
 };
@@ -115,7 +116,8 @@ export function favourite(status) {
     dispatch(favouriteRequest(status));
 
     api(getState).post(`/api/v1/statuses/${status.get('id')}/favourite`).then(function (response) {
-      dispatch(favouriteSuccess(status, response.data));
+      dispatch(importFetchedStatus(response.data));
+      dispatch(favouriteSuccess(status));
     }).catch(function (error) {
       dispatch(favouriteFail(status, error));
     });
@@ -127,7 +129,8 @@ export function unfavourite(status) {
     dispatch(unfavouriteRequest(status));
 
     api(getState).post(`/api/v1/statuses/${status.get('id')}/unfavourite`).then(response => {
-      dispatch(unfavouriteSuccess(status, response.data));
+      dispatch(importFetchedStatus(response.data));
+      dispatch(unfavouriteSuccess(status));
     }).catch(error => {
       dispatch(unfavouriteFail(status, error));
     });
@@ -142,11 +145,10 @@ export function favouriteRequest(status) {
   };
 };
 
-export function favouriteSuccess(status, response) {
+export function favouriteSuccess(status) {
   return {
     type: FAVOURITE_SUCCESS,
     status: status,
-    response: response,
     skipLoading: true,
   };
 };
@@ -168,11 +170,10 @@ export function unfavouriteRequest(status) {
   };
 };
 
-export function unfavouriteSuccess(status, response) {
+export function unfavouriteSuccess(status) {
   return {
     type: UNFAVOURITE_SUCCESS,
     status: status,
-    response: response,
     skipLoading: true,
   };
 };
@@ -191,6 +192,7 @@ export function fetchReblogs(id) {
     dispatch(fetchReblogsRequest(id));
 
     api(getState).get(`/api/v1/statuses/${id}/reblogged_by`).then(response => {
+      dispatch(importFetchedAccounts(response.data));
       dispatch(fetchReblogsSuccess(id, response.data));
     }).catch(error => {
       dispatch(fetchReblogsFail(id, error));
@@ -225,6 +227,7 @@ export function fetchFavourites(id) {
     dispatch(fetchFavouritesRequest(id));
 
     api(getState).get(`/api/v1/statuses/${id}/favourited_by`).then(response => {
+      dispatch(importFetchedAccounts(response.data));
       dispatch(fetchFavouritesSuccess(id, response.data));
     }).catch(error => {
       dispatch(fetchFavouritesFail(id, error));
@@ -259,7 +262,8 @@ export function pin(status) {
     dispatch(pinRequest(status));
 
     api(getState).post(`/api/v1/statuses/${status.get('id')}/pin`).then(response => {
-      dispatch(pinSuccess(status, response.data));
+      dispatch(importFetchedStatus(response.data));
+      dispatch(pinSuccess(status));
     }).catch(error => {
       dispatch(pinFail(status, error));
     });
@@ -274,11 +278,10 @@ export function pinRequest(status) {
   };
 };
 
-export function pinSuccess(status, response) {
+export function pinSuccess(status) {
   return {
     type: PIN_SUCCESS,
     status,
-    response,
     skipLoading: true,
   };
 };
@@ -297,7 +300,8 @@ export function unpin (status) {
     dispatch(unpinRequest(status));
 
     api(getState).post(`/api/v1/statuses/${status.get('id')}/unpin`).then(response => {
-      dispatch(unpinSuccess(status, response.data));
+      dispatch(importFetchedStatus(response.data));
+      dispatch(unpinSuccess(status));
     }).catch(error => {
       dispatch(unpinFail(status, error));
     });
@@ -312,11 +316,10 @@ export function unpinRequest(status) {
   };
 };
 
-export function unpinSuccess(status, response) {
+export function unpinSuccess(status) {
   return {
     type: UNPIN_SUCCESS,
     status,
-    response,
     skipLoading: true,
   };
 };

app/javascript/mastodon/actions/lists.js

@@ -1,4 +1,6 @@
 import api from '../api';
+import { importFetchedAccounts } from './importer';
+import { showAlertForError } from './alerts';
 
 export const LIST_FETCH_REQUEST = 'LIST_FETCH_REQUEST';
 export const LIST_FETCH_SUCCESS = 'LIST_FETCH_SUCCESS';
@@ -200,9 +202,10 @@ export const deleteListFail = (id, error) => ({
 export const fetchListAccounts = listId => (dispatch, getState) => {
   dispatch(fetchListAccountsRequest(listId));
 
-  api(getState).get(`/api/v1/lists/${listId}/accounts`, { params: { limit: 0 } })
-    .then(({ data }) => dispatch(fetchListAccountsSuccess(listId, data)))
-    .catch(err => dispatch(fetchListAccountsFail(listId, err)));
+  api(getState).get(`/api/v1/lists/${listId}/accounts`, { params: { limit: 0 } }).then(({ data }) => {
+    dispatch(importFetchedAccounts(data));
+    dispatch(fetchListAccountsSuccess(listId, data));
+  }).catch(err => dispatch(fetchListAccountsFail(listId, err)));
 };
 
 export const fetchListAccountsRequest = id => ({
@@ -231,8 +234,10 @@ export const fetchListSuggestions = q => (dispatch, getState) => {
     following: true,
   };
 
-  api(getState).get('/api/v1/accounts/search', { params })
-    .then(({ data }) => dispatch(fetchListSuggestionsReady(q, data)));
+  api(getState).get('/api/v1/accounts/search', { params }).then(({ data }) => {
+    dispatch(importFetchedAccounts(data));
+    dispatch(fetchListSuggestionsReady(q, data));
+  }).catch(error => dispatch(showAlertForError(error)));
 };
 
 export const fetchListSuggestionsReady = (query, accounts) => ({

app/javascript/mastodon/actions/mutes.js

@@ -1,5 +1,6 @@
 import api, { getLinks } from '../api';
 import { fetchRelationships } from './accounts';
+import { importFetchedAccounts } from './importer';
 import { openModal } from './modal';
 
 export const MUTES_FETCH_REQUEST = 'MUTES_FETCH_REQUEST';
@@ -19,6 +20,7 @@ export function fetchMutes() {
 
     api(getState).get('/api/v1/mutes').then(response => {
       const next = getLinks(response).refs.find(link => link.rel === 'next');
+      dispatch(importFetchedAccounts(response.data));
       dispatch(fetchMutesSuccess(response.data, next ? next.uri : null));
       dispatch(fetchRelationships(response.data.map(item => item.id)));
     }).catch(error => dispatch(fetchMutesFail(error)));
@@ -58,6 +60,7 @@ export function expandMutes() {
 
     api(getState).get(url).then(response => {
       const next = getLinks(response).refs.find(link => link.rel === 'next');
+      dispatch(importFetchedAccounts(response.data));
       dispatch(expandMutesSuccess(response.data, next ? next.uri : null));
       dispatch(fetchRelationships(response.data.map(item => item.id)));
     }).catch(error => dispatch(expandMutesFail(error)));

app/javascript/mastodon/actions/notifications.js

@@ -1,14 +1,17 @@
 import api, { getLinks } from '../api';
-import { List as ImmutableList } from 'immutable';
 import IntlMessageFormat from 'intl-messageformat';
 import { fetchRelationships } from './accounts';
+import {
+  importFetchedAccount,
+  importFetchedAccounts,
+  importFetchedStatus,
+  importFetchedStatuses,
+} from './importer';
 import { defineMessages } from 'react-intl';
+import { unescapeHTML } from '../utils/html';
 
-export const NOTIFICATIONS_UPDATE = 'NOTIFICATIONS_UPDATE';
-
-export const NOTIFICATIONS_REFRESH_REQUEST = 'NOTIFICATIONS_REFRESH_REQUEST';
-export const NOTIFICATIONS_REFRESH_SUCCESS = 'NOTIFICATIONS_REFRESH_SUCCESS';
-export const NOTIFICATIONS_REFRESH_FAIL    = 'NOTIFICATIONS_REFRESH_FAIL';
+export const NOTIFICATIONS_UPDATE      = 'NOTIFICATIONS_UPDATE';
+export const NOTIFICATIONS_UPDATE_NOOP = 'NOTIFICATIONS_UPDATE_NOOP';
 
 export const NOTIFICATIONS_EXPAND_REQUEST = 'NOTIFICATIONS_EXPAND_REQUEST';
 export const NOTIFICATIONS_EXPAND_SUCCESS = 'NOTIFICATIONS_EXPAND_SUCCESS';
@@ -19,6 +22,7 @@ export const NOTIFICATIONS_SCROLL_TOP = 'NOTIFICATIONS_SCROLL_TOP';
 
 defineMessages({
   mention: { id: 'notification.mention', defaultMessage: '{name} mentioned you' },
+  group: { id: 'notifications.group', defaultMessage: '{count} notifications' },
 });
 
 const fetchRelatedRelationships = (dispatch, notifications) => {
@@ -29,27 +33,32 @@ const fetchRelatedRelationships = (dispatch, notifications) => {
   }
 };
 
-const unescapeHTML = (html) => {
-  const wrapper = document.createElement('div');
-  html = html.replace(/<br \/>|<br>|\n/g, ' ');
-  wrapper.innerHTML = html;
-  return wrapper.textContent;
-};
-
 export function updateNotifications(notification, intlMessages, intlLocale) {
   return (dispatch, getState) => {
-    const showAlert = getState().getIn(['settings', 'notifications', 'alerts', notification.type], true);
-    const playSound = getState().getIn(['settings', 'notifications', 'sounds', notification.type], true);
+    const showInColumn = getState().getIn(['settings', 'notifications', 'shows', notification.type], true);
+    const showAlert    = getState().getIn(['settings', 'notifications', 'alerts', notification.type], true);
+    const playSound    = getState().getIn(['settings', 'notifications', 'sounds', notification.type], true);
 
-    dispatch({
-      type: NOTIFICATIONS_UPDATE,
-      notification,
-      account: notification.account,
-      status: notification.status,
-      meta: playSound ? { sound: 'boop' } : undefined,
-    });
+    if (showInColumn) {
+      dispatch(importFetchedAccount(notification.account));
+
+      if (notification.status) {
+        dispatch(importFetchedStatus(notification.status));
+      }
 
-    fetchRelatedRelationships(dispatch, [notification]);
+      dispatch({
+        type: NOTIFICATIONS_UPDATE,
+        notification,
+        meta: playSound ? { sound: 'boop' } : undefined,
+      });
+
+      fetchRelatedRelationships(dispatch, [notification]);
+    } else if (playSound) {
+      dispatch({
+        type: NOTIFICATIONS_UPDATE_NOOP,
+        meta: { sound: 'boop' },
+      });
+    }
 
     // Desktop notifications
     if (typeof window.Notification !== 'undefined' && showAlert) {
@@ -57,6 +66,7 @@ export function updateNotifications(notification, intlMessages, intlLocale) {
       const body  = (notification.status && notification.status.spoiler_text.length > 0) ? notification.status.spoiler_text : unescapeHTML(notification.status ? notification.status.content : '');
 
       const notify = new Notification(title, { body, icon: notification.account.avatar, tag: notification.id });
+
       notify.addEventListener('click', () => {
         window.focus();
         notify.close();
@@ -67,84 +77,40 @@ export function updateNotifications(notification, intlMessages, intlLocale) {
 
 const excludeTypesFromSettings = state => state.getIn(['settings', 'notifications', 'shows']).filter(enabled => !enabled).keySeq().toJS();
 
-export function refreshNotifications() {
-  return (dispatch, getState) => {
-    const params = {};
-    const ids    = getState().getIn(['notifications', 'items']);
-
-    let skipLoading = false;
-
-    if (ids.size > 0) {
-      params.since_id = ids.first().get('id');
-    }
-
-    if (getState().getIn(['notifications', 'loaded'])) {
-      skipLoading = true;
-    }
-
-    params.exclude_types = excludeTypesFromSettings(getState());
+const noOp = () => {};
 
-    dispatch(refreshNotificationsRequest(skipLoading));
-
-    api(getState).get('/api/v1/notifications', { params }).then(response => {
-      const next = getLinks(response).refs.find(link => link.rel === 'next');
-
-      dispatch(refreshNotificationsSuccess(response.data, skipLoading, next ? next.uri : null));
-      fetchRelatedRelationships(dispatch, response.data);
-    }).catch(error => {
-      dispatch(refreshNotificationsFail(error, skipLoading));
-    });
-  };
-};
-
-export function refreshNotificationsRequest(skipLoading) {
-  return {
-    type: NOTIFICATIONS_REFRESH_REQUEST,
-    skipLoading,
-  };
-};
-
-export function refreshNotificationsSuccess(notifications, skipLoading, next) {
-  return {
-    type: NOTIFICATIONS_REFRESH_SUCCESS,
-    notifications,
-    accounts: notifications.map(item => item.account),
-    statuses: notifications.map(item => item.status).filter(status => !!status),
-    skipLoading,
-    next,
-  };
-};
-
-export function refreshNotificationsFail(error, skipLoading) {
-  return {
-    type: NOTIFICATIONS_REFRESH_FAIL,
-    error,
-    skipLoading,
-  };
-};
-
-export function expandNotifications() {
+export function expandNotifications({ maxId } = {}, done = noOp) {
   return (dispatch, getState) => {
-    const items  = getState().getIn(['notifications', 'items'], ImmutableList());
+    const notifications = getState().get('notifications');
 
-    if (getState().getIn(['notifications', 'isLoading']) || items.size === 0) {
+    if (notifications.get('isLoading')) {
+      done();
       return;
     }
 
     const params = {
-      max_id: items.last().get('id'),
-      limit: 20,
+      max_id: maxId,
       exclude_types: excludeTypesFromSettings(getState()),
     };
 
+    if (!maxId && notifications.get('items').size > 0) {
+      params.since_id = notifications.getIn(['items', 0]);
+    }
+
     dispatch(expandNotificationsRequest());
 
     api(getState).get('/api/v1/notifications', { params }).then(response => {
       const next = getLinks(response).refs.find(link => link.rel === 'next');
+
+      dispatch(importFetchedAccounts(response.data.map(item => item.account)));
+      dispatch(importFetchedStatuses(response.data.map(item => item.status).filter(status => !!status)));
+
       dispatch(expandNotificationsSuccess(response.data, next ? next.uri : null));
       fetchRelatedRelationships(dispatch, response.data);
+      done();
     }).catch(error => {
       dispatch(expandNotificationsFail(error));
+      done();
     });
   };
 };
@@ -159,8 +125,6 @@ export function expandNotificationsSuccess(notifications, next) {
   return {
     type: NOTIFICATIONS_EXPAND_SUCCESS,
     notifications,
-    accounts: notifications.map(item => item.account),
-    statuses: notifications.map(item => item.status).filter(status => !!status),
     next,
   };
 };

app/javascript/mastodon/actions/pin_statuses.js

@@ -1,4 +1,5 @@
 import api from '../api';
+import { importFetchedStatuses } from './importer';
 
 export const PINNED_STATUSES_FETCH_REQUEST = 'PINNED_STATUSES_FETCH_REQUEST';
 export const PINNED_STATUSES_FETCH_SUCCESS = 'PINNED_STATUSES_FETCH_SUCCESS';
@@ -11,6 +12,7 @@ export function fetchPinnedStatuses() {
     dispatch(fetchPinnedStatusesRequest());
 
     api(getState).get(`/api/v1/accounts/${me}/statuses`, { params: { pinned: true } }).then(response => {
+      dispatch(importFetchedStatuses(response.data));
       dispatch(fetchPinnedStatusesSuccess(response.data, null));
     }).catch(error => {
       dispatch(fetchPinnedStatusesFail(error));

app/javascript/mastodon/actions/push_notifications/registerer.js

@@ -1,4 +1,5 @@
 import api from '../../api';
+import { decode as decodeBase64 } from '../../utils/base64';
 import { pushNotificationsSetting } from '../../settings';
 import { setBrowserSupport, setSubscription, clearSubscription } from './setter';
 import { me } from '../../initial_state';
@@ -10,13 +11,7 @@ const urlBase64ToUint8Array = (base64String) => {
     .replace(/\-/g, '+')
     .replace(/_/g, '/');
 
-  const rawData = window.atob(base64);
-  const outputArray = new Uint8Array(rawData.length);
-
-  for (let i = 0; i < rawData.length; ++i) {
-    outputArray[i] = rawData.charCodeAt(i);
-  }
-  return outputArray;
+  return decodeBase64(base64);
 };
 
 const getApplicationServerKey = () => document.querySelector('[name="applicationServerKey"]').getAttribute('content');
@@ -36,7 +31,7 @@ const subscribe = (registration) =>
 const unsubscribe = ({ registration, subscription }) =>
   subscription ? subscription.unsubscribe().then(() => registration) : registration;
 
-const sendSubscriptionToBackend = (getState, subscription) => {
+const sendSubscriptionToBackend = (subscription) => {
   const params = { subscription };
 
   if (me) {
@@ -46,7 +41,7 @@ const sendSubscriptionToBackend = (getState, subscription) => {
     }
   }
 
-  return api(getState).post('/api/web/push_subscriptions', params).then(response => response.data);
+  return api().post('/api/web/push_subscriptions', params).then(response => response.data);
 };
 
 // Last one checks for payload support: https://web-push-book.gauntface.com/chapter-06/01-non-standards-browsers/#no-payload
@@ -56,13 +51,6 @@ export function register () {
   return (dispatch, getState) => {
     dispatch(setBrowserSupport(supportsPushNotifications));
 
-    if (me && !pushNotificationsSetting.get(me)) {
-      const alerts = getState().getIn(['push_notifications', 'alerts']);
-      if (alerts) {
-        pushNotificationsSetting.set(me, { alerts: alerts });
-      }
-    }
-
     if (supportsPushNotifications) {
       if (!getApplicationServerKey()) {
         console.error('The VAPID public key is not set. You will not be able to receive Web Push Notifications.');
@@ -85,13 +73,13 @@ export function register () {
             } else {
               // Something went wrong, try to subscribe again
               return unsubscribe({ registration, subscription }).then(subscribe).then(
-                subscription => sendSubscriptionToBackend(getState, subscription));
+                subscription => sendSubscriptionToBackend(subscription));
             }
           }
 
           // No subscription, try to subscribe
           return subscribe(registration).then(
-            subscription => sendSubscriptionToBackend(getState, subscription));
+            subscription => sendSubscriptionToBackend(subscription));
         })
         .then(subscription => {
           // If we got a PushSubscription (and not a subscription object from the backend)
@@ -116,14 +104,11 @@ export function register () {
             pushNotificationsSetting.remove(me);
           }
 
-          try {
-            getRegistration()
-              .then(getPushSubscription)
-              .then(unsubscribe);
-          } catch (e) {
-
-          }
-        });
+          return getRegistration()
+            .then(getPushSubscription)
+            .then(unsubscribe);
+        })
+        .catch(console.warn);
     } else {
       console.warn('Your browser does not support Web Push Notifications.');
     }
@@ -137,12 +122,12 @@ export function saveSettings() {
     const alerts = state.get('alerts');
     const data = { alerts };
 
-    api(getState).put(`/api/web/push_subscriptions/${subscription.get('id')}`, {
+    api().put(`/api/web/push_subscriptions/${subscription.get('id')}`, {
       data,
     }).then(() => {
       if (me) {
         pushNotificationsSetting.set(me, data);
       }
-    });
+    }).catch(console.warn);
   };
 }

app/javascript/mastodon/actions/search.js

@@ -1,5 +1,6 @@
 import api from '../api';
 import { fetchRelationships } from './accounts';
+import { importFetchedAccounts, importFetchedStatuses } from './importer';
 
 export const SEARCH_CHANGE = 'SEARCH_CHANGE';
 export const SEARCH_CLEAR  = 'SEARCH_CLEAR';
@@ -38,6 +39,14 @@ export function submitSearch() {
         resolve: true,
       },
     }).then(response => {
+      if (response.data.accounts) {
+        dispatch(importFetchedAccounts(response.data.accounts));
+      }
+
+      if (response.data.statuses) {
+        dispatch(importFetchedStatuses(response.data.statuses));
+      }
+
       dispatch(fetchSearchSuccess(response.data));
       dispatch(fetchRelationships(response.data.accounts.map(item => item.id)));
     }).catch(error => {
@@ -56,8 +65,6 @@ export function fetchSearchSuccess(results) {
   return {
     type: SEARCH_FETCH_SUCCESS,
     results,
-    accounts: results.accounts,
-    statuses: results.statuses,
   };
 };
 

app/javascript/mastodon/actions/settings.js

@@ -1,5 +1,6 @@
 import api from '../api';
 import { debounce } from 'lodash';
+import { showAlertForError } from './alerts';
 
 export const SETTING_CHANGE = 'SETTING_CHANGE';
 export const SETTING_SAVE   = 'SETTING_SAVE';
@@ -23,7 +24,9 @@ const debouncedSave = debounce((dispatch, getState) => {
 
   const data = getState().get('settings').filter((_, path) => path !== 'saved').toJS();
 
-  api(getState).put('/api/web/settings', { data }).then(() => dispatch({ type: SETTING_SAVE }));
+  api().put('/api/web/settings', { data })
+    .then(() => dispatch({ type: SETTING_SAVE }))
+    .catch(error => dispatch(showAlertForError(error)));
 }, 5000, { trailing: true });
 
 export function saveSettings() {

app/javascript/mastodon/actions/statuses.js

@@ -1,7 +1,10 @@
 import api from '../api';
+import openDB from '../storage/db';
+import { evictStatus } from '../storage/modifier';
 
 import { deleteFromTimelines } from './timelines';
 import { fetchStatusCard } from './cards';
+import { importFetchedStatus, importFetchedStatuses, importAccount, importStatus } from './importer';
 
 export const STATUS_FETCH_REQUEST = 'STATUS_FETCH_REQUEST';
 export const STATUS_FETCH_SUCCESS = 'STATUS_FETCH_SUCCESS';
@@ -34,6 +37,48 @@ export function fetchStatusRequest(id, skipLoading) {
   };
 };
 
+function getFromDB(dispatch, getState, accountIndex, index, id) {
+  return new Promise((resolve, reject) => {
+    const request = index.get(id);
+
+    request.onerror = reject;
+
+    request.onsuccess = () => {
+      const promises = [];
+
+      if (!request.result) {
+        reject();
+        return;
+      }
+
+      dispatch(importStatus(request.result));
+
+      if (getState().getIn(['accounts', request.result.account], null) === null) {
+        promises.push(new Promise((accountResolve, accountReject) => {
+          const accountRequest = accountIndex.get(request.result.account);
+
+          accountRequest.onerror = accountReject;
+          accountRequest.onsuccess = () => {
+            if (!request.result) {
+              accountReject();
+              return;
+            }
+
+            dispatch(importAccount(accountRequest.result));
+            accountResolve();
+          };
+        }));
+      }
+
+      if (request.result.reblog && getState().getIn(['statuses', request.result.reblog], null) === null) {
+        promises.push(getFromDB(dispatch, getState, accountIndex, index, request.result.reblog));
+      }
+
+      resolve(Promise.all(promises));
+    };
+  });
+}
+
 export function fetchStatus(id) {
   return (dispatch, getState) => {
     const skipLoading = getState().getIn(['statuses', id], null) !== null;
@@ -47,18 +92,31 @@ export function fetchStatus(id) {
 
     dispatch(fetchStatusRequest(id, skipLoading));
 
-    api(getState).get(`/api/v1/statuses/${id}`).then(response => {
-      dispatch(fetchStatusSuccess(response.data, skipLoading));
-    }).catch(error => {
+    openDB().then(db => {
+      const transaction = db.transaction(['accounts', 'statuses'], 'read');
+      const accountIndex = transaction.objectStore('accounts').index('id');
+      const index = transaction.objectStore('statuses').index('id');
+
+      return getFromDB(dispatch, getState, accountIndex, index, id).then(() => {
+        db.close();
+      }, error => {
+        db.close();
+        throw error;
+      });
+    }).then(() => {
+      dispatch(fetchStatusSuccess(skipLoading));
+    }, () => api(getState).get(`/api/v1/statuses/${id}`).then(response => {
+      dispatch(importFetchedStatus(response.data));
+      dispatch(fetchStatusSuccess(skipLoading));
+    })).catch(error => {
       dispatch(fetchStatusFail(id, error, skipLoading));
     });
   };
 };
 
-export function fetchStatusSuccess(status, skipLoading) {
+export function fetchStatusSuccess(skipLoading) {
   return {
     type: STATUS_FETCH_SUCCESS,
-    status,
     skipLoading,
   };
 };
@@ -78,6 +136,7 @@ export function deleteStatus(id) {
     dispatch(deleteStatusRequest(id));
 
     api(getState).delete(`/api/v1/statuses/${id}`).then(() => {
+      evictStatus(id);
       dispatch(deleteStatusSuccess(id));
       dispatch(deleteFromTimelines(id));
     }).catch(error => {
@@ -113,6 +172,7 @@ export function fetchContext(id) {
     dispatch(fetchContextRequest(id));
 
     api(getState).get(`/api/v1/statuses/${id}/context`).then(response => {
+      dispatch(importFetchedStatuses(response.data.ancestors.concat(response.data.descendants)));
       dispatch(fetchContextSuccess(id, response.data.ancestors, response.data.descendants));
 
     }).catch(error => {