diff --git a/.github/workflows/test-ruby.yml b/.github/workflows/test-ruby.yml new file mode 100644 index 00000000000000..90467e46d3f50a --- /dev/null +++ b/.github/workflows/test-ruby.yml @@ -0,0 +1,151 @@ +name: Ruby Testing + +on: + push: + branches-ignore: + - 'dependabot/**' + - 'renovate/**' + pull_request: + +env: + BUNDLE_CLEAN: true + BUNDLE_FROZEN: true + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + + strategy: + fail-fast: true + matrix: + mode: + - production + - test + env: + RAILS_ENV: ${{ matrix.mode }} + BUNDLE_WITH: ${{ matrix.mode }} + OTP_SECRET: precompile_placeholder + SECRET_KEY_BASE: precompile_placeholder + + steps: + - uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v3 + with: + cache: yarn + node-version-file: '.nvmrc' + + - name: Install native Ruby dependencies + run: | + sudo apt-get update + sudo apt-get install -y libicu-dev libidn11-dev + + - name: Set up bundler cache + uses: ruby/setup-ruby@v1 + with: + ruby-version: .ruby-version + bundler-cache: true + + - run: yarn --frozen-lockfile --production + - name: Precompile assets + # Previously had set this, but it's not supported + # export NODE_OPTIONS=--openssl-legacy-provider + run: |- + ./bin/rails assets:precompile + + - uses: actions/upload-artifact@v3 + if: matrix.mode == 'test' + with: + path: |- + ./public/assets + ./public/packs-test + name: ${{ github.sha }} + retention-days: 0 + + test: + runs-on: ubuntu-latest + + needs: + - build + + services: + postgres: + image: postgres:14-alpine + env: + POSTGRES_PASSWORD: postgres + POSTGRES_USER: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + redis: + image: redis:7-alpine + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 6379:6379 + + env: + DB_HOST: localhost + DB_USER: postgres + DB_PASS: postgres + DISABLE_SIMPLECOV: true + RAILS_ENV: test + ALLOW_NOPAM: true + PAM_ENABLED: true + PAM_DEFAULT_SERVICE: pam_test + PAM_CONTROLLED_SERVICE: pam_test_controlled + OIDC_ENABLED: true + OIDC_SCOPE: read + SAML_ENABLED: true + CAS_ENABLED: true + BUNDLE_WITH: 'pam_authentication test' + CI_JOBS: ${{ matrix.ci_job }}/4 + + strategy: + fail-fast: false + matrix: + ruby-version: + - '.ruby-version' + ci_job: + - 1 + - 2 + - 3 + - 4 + steps: + - uses: actions/checkout@v4 + + - uses: actions/download-artifact@v3 + with: + path: './public' + name: ${{ github.sha }} + + - name: Update package index + run: sudo apt-get update + + - name: Install native Ruby dependencies + run: sudo apt-get install -y libicu-dev libidn11-dev + + - name: Install additional system dependencies + run: sudo apt-get install -y ffmpeg imagemagick libpam-dev + + - name: Set up bundler cache + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby-version}} + bundler-cache: true + + - name: Load database schema + run: './bin/rails db:create db:schema:load db:seed' diff --git a/CHANGELOG.md b/CHANGELOG.md index 57fabea9f21913..e65ebd8f96247b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,26 @@ Changelog All notable changes to this project will be documented in this file. +## [4.1.16] - 2024-02-23 + +### Added + +- Add hourly task to automatically require approval for new registrations in the absence of moderators ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29318), [ClearlyClaire](https://github.com/mastodon/mastodon/pull/29355)) + In order to prevent future abandoned Mastodon servers from being used for spam, harassment and other malicious activity, Mastodon will now automatically switch new user registrations to require moderator approval whenever they are left open and no activity (including non-moderation actions from apps) from any logged-in user with permission to access moderation reports has been detected in a full week. + When this happens, users with the permission to change server settings will receive an email notification. + This feature is disabled when `EMAIL_DOMAIN_ALLOWLIST` is used, and can also be disabled with `DISABLE_AUTOMATIC_SWITCHING_TO_APPROVED_REGISTRATIONS=true`. + +### Changed + +- Change registrations to be closed by default on new installations ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29280)) + If you are running a server and never changed your registrations mode from the default, updating will automatically close your registrations. + Simply re-enable them through the administration interface or using `tootctl settings registrations open` if you want to enable them again. + +### Fixed + +- Fix processing of remote ActivityPub actors making use of `Link` objects as `Image` `url` ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29335)) +- Fix link verifications when page size exceeds 1MB ([ClearlyClaire](https://github.com/mastodon/mastodon/pull/29358)) + ## [4.1.15] - 2024-02-16 ### Fixed diff --git a/Gemfile.lock b/Gemfile.lock index ef1a19ae129bc8..17700ba1e49876 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -10,40 +10,40 @@ GIT GEM remote: https://rubygems.org/ specs: - actioncable (6.1.7.6) - actionpack (= 6.1.7.6) - activesupport (= 6.1.7.6) + actioncable (6.1.7.7) + actionpack (= 6.1.7.7) + activesupport (= 6.1.7.7) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailbox (6.1.7.6) - actionpack (= 6.1.7.6) - activejob (= 6.1.7.6) - activerecord (= 6.1.7.6) - activestorage (= 6.1.7.6) - activesupport (= 6.1.7.6) + actionmailbox (6.1.7.7) + actionpack (= 6.1.7.7) + activejob (= 6.1.7.7) + activerecord (= 6.1.7.7) + activestorage (= 6.1.7.7) + activesupport (= 6.1.7.7) mail (>= 2.7.1) - actionmailer (6.1.7.6) - actionpack (= 6.1.7.6) - actionview (= 6.1.7.6) - activejob (= 6.1.7.6) - activesupport (= 6.1.7.6) + actionmailer (6.1.7.7) + actionpack (= 6.1.7.7) + actionview (= 6.1.7.7) + activejob (= 6.1.7.7) + activesupport (= 6.1.7.7) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (6.1.7.6) - actionview (= 6.1.7.6) - activesupport (= 6.1.7.6) + actionpack (6.1.7.7) + actionview (= 6.1.7.7) + activesupport (= 6.1.7.7) rack (~> 2.0, >= 2.0.9) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.2.0) - actiontext (6.1.7.6) - actionpack (= 6.1.7.6) - activerecord (= 6.1.7.6) - activestorage (= 6.1.7.6) - activesupport (= 6.1.7.6) + actiontext (6.1.7.7) + actionpack (= 6.1.7.7) + activerecord (= 6.1.7.7) + activestorage (= 6.1.7.7) + activesupport (= 6.1.7.7) nokogiri (>= 1.8.5) - actionview (6.1.7.6) - activesupport (= 6.1.7.6) + actionview (6.1.7.7) + activesupport (= 6.1.7.7) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) @@ -54,22 +54,22 @@ GEM case_transform (>= 0.2) jsonapi-renderer (>= 0.1.1.beta1, < 0.3) active_record_query_trace (1.8) - activejob (6.1.7.6) - activesupport (= 6.1.7.6) + activejob (6.1.7.7) + activesupport (= 6.1.7.7) globalid (>= 0.3.6) - activemodel (6.1.7.6) - activesupport (= 6.1.7.6) - activerecord (6.1.7.6) - activemodel (= 6.1.7.6) - activesupport (= 6.1.7.6) - activestorage (6.1.7.6) - actionpack (= 6.1.7.6) - activejob (= 6.1.7.6) - activerecord (= 6.1.7.6) - activesupport (= 6.1.7.6) + activemodel (6.1.7.7) + activesupport (= 6.1.7.7) + activerecord (6.1.7.7) + activemodel (= 6.1.7.7) + activesupport (= 6.1.7.7) + activestorage (6.1.7.7) + actionpack (= 6.1.7.7) + activejob (= 6.1.7.7) + activerecord (= 6.1.7.7) + activesupport (= 6.1.7.7) marcel (~> 1.0) mini_mime (>= 1.1.0) - activesupport (6.1.7.6) + activesupport (6.1.7.7) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) @@ -173,7 +173,7 @@ GEM cocoon (1.2.15) coderay (1.1.3) color_diff (0.1) - concurrent-ruby (1.2.2) + concurrent-ruby (1.2.3) connection_pool (2.3.0) cose (1.2.1) cbor (~> 0.5.9) @@ -183,7 +183,7 @@ GEM crass (1.0.6) css_parser (1.12.0) addressable - date (3.3.3) + date (3.3.4) debug_inspector (1.0.0) devise (4.8.1) bcrypt (~> 3.0) @@ -416,11 +416,11 @@ GEM net-ldap (0.17.1) net-pop (0.1.2) net-protocol - net-protocol (0.2.1) + net-protocol (0.2.2) timeout net-scp (4.0.0.rc1) net-ssh (>= 2.6.5, < 8.0.0) - net-smtp (0.3.3) + net-smtp (0.3.4) net-protocol net-ssh (7.0.1) nio4r (2.5.9) @@ -497,7 +497,7 @@ GEM activesupport (>= 3.0.0) raabro (1.4.0) racc (1.7.3) - rack (2.2.8) + rack (2.2.8.1) rack-attack (6.6.1) rack (>= 1.0, < 3) rack-cors (1.1.1) @@ -512,20 +512,20 @@ GEM rack rack-test (2.0.2) rack (>= 1.3) - rails (6.1.7.6) - actioncable (= 6.1.7.6) - actionmailbox (= 6.1.7.6) - actionmailer (= 6.1.7.6) - actionpack (= 6.1.7.6) - actiontext (= 6.1.7.6) - actionview (= 6.1.7.6) - activejob (= 6.1.7.6) - activemodel (= 6.1.7.6) - activerecord (= 6.1.7.6) - activestorage (= 6.1.7.6) - activesupport (= 6.1.7.6) + rails (6.1.7.7) + actioncable (= 6.1.7.7) + actionmailbox (= 6.1.7.7) + actionmailer (= 6.1.7.7) + actionpack (= 6.1.7.7) + actiontext (= 6.1.7.7) + actionview (= 6.1.7.7) + activejob (= 6.1.7.7) + activemodel (= 6.1.7.7) + activerecord (= 6.1.7.7) + activestorage (= 6.1.7.7) + activesupport (= 6.1.7.7) bundler (>= 1.15.0) - railties (= 6.1.7.6) + railties (= 6.1.7.7) sprockets-rails (>= 2.0.0) rails-controller-testing (1.0.5) actionpack (>= 5.0.1.rc1) @@ -541,9 +541,9 @@ GEM railties (>= 6.0.0, < 7) rails-settings-cached (0.6.6) rails (>= 4.2.0) - railties (6.1.7.6) - actionpack (= 6.1.7.6) - activesupport (= 6.1.7.6) + railties (6.1.7.7) + actionpack (= 6.1.7.7) + activesupport (= 6.1.7.7) method_source rake (>= 12.2) thor (~> 1.0) @@ -753,7 +753,7 @@ GEM xorcist (1.1.3) xpath (3.2.0) nokogiri (~> 1.8) - zeitwerk (2.6.12) + zeitwerk (2.6.13) PLATFORMS ruby diff --git a/app/mailers/admin_mailer.rb b/app/mailers/admin_mailer.rb index bc6d87ae6fc11c..c889c91d6e2a28 100644 --- a/app/mailers/admin_mailer.rb +++ b/app/mailers/admin_mailer.rb @@ -47,4 +47,13 @@ def new_trends(recipient, links, tags, statuses) mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trends.subject', instance: @instance) end end + + def auto_close_registrations(recipient) + @me = recipient + @instance = Rails.configuration.x.local_domain + + locale_for_account(@me) do + mail to: @me.user_email, subject: I18n.t('admin_mailer.auto_close_registrations.subject', instance: @instance) + end + end end diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb index fef3781e1aaca1..d2223938dc3045 100644 --- a/app/services/activitypub/process_account_service.rb +++ b/app/services/activitypub/process_account_service.rb @@ -199,10 +199,15 @@ def image_url(key) value = first_of_value(@json[key]) return if value.nil? - return value['url'] if value.is_a?(Hash) - image = fetch_resource_without_id_validation(value) - image['url'] if image + if value.is_a?(String) + value = fetch_resource_without_id_validation(value) + return if value.nil? + end + + value = first_of_value(value['url']) if value.is_a?(Hash) && value['type'] == 'Image' + value = value['href'] if value.is_a?(Hash) + value if value.is_a?(String) end def public_key diff --git a/app/services/verify_link_service.rb b/app/services/verify_link_service.rb index 707aeb4e08b47a..b317fc31a859da 100644 --- a/app/services/verify_link_service.rb +++ b/app/services/verify_link_service.rb @@ -19,7 +19,7 @@ def call(field) def perform_request! @body = Request.new(:get, @url).add_headers('Accept' => 'text/html').perform do |res| - res.code == 200 ? res.body_with_limit : nil + res.code == 200 ? res.truncated_body : nil end end diff --git a/app/views/admin/settings/registrations/show.html.haml b/app/views/admin/settings/registrations/show.html.haml index 0db9f3536fb9e9..c657a7f6f57dff 100644 --- a/app/views/admin/settings/registrations/show.html.haml +++ b/app/views/admin/settings/registrations/show.html.haml @@ -13,6 +13,8 @@ %p.lead= t('admin.settings.registrations.preamble') + .flash-message= t('admin.settings.registrations.moderation_recommandation') + .fields-row .fields-row__column.fields-row__column-6.fields-group = f.input :registrations_mode, collection: %w(open approved none), wrapper: :with_label, include_blank: false, label_method: lambda { |mode| I18n.t("admin.settings.registrations_mode.modes.#{mode}") } diff --git a/app/views/admin_mailer/auto_close_registrations.text.erb b/app/views/admin_mailer/auto_close_registrations.text.erb new file mode 100644 index 00000000000000..c0f848692965b9 --- /dev/null +++ b/app/views/admin_mailer/auto_close_registrations.text.erb @@ -0,0 +1,3 @@ +<%= raw t('admin_mailer.auto_close_registrations.body', instance: @instance) %> + +<%= raw t('application_mailer.view')%> <%= admin_settings_registrations_url %> diff --git a/app/workers/scheduler/auto_close_registrations_scheduler.rb b/app/workers/scheduler/auto_close_registrations_scheduler.rb new file mode 100644 index 00000000000000..f774dda117880f --- /dev/null +++ b/app/workers/scheduler/auto_close_registrations_scheduler.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +class Scheduler::AutoCloseRegistrationsScheduler + include Sidekiq::Worker + include Redisable + + sidekiq_options retry: 0 + + # Automatically switch away from open registrations if no + # moderator had any activity in that period of time + OPEN_REGISTRATIONS_MODERATOR_THRESHOLD = 1.week + UserTrackingConcern::SIGN_IN_UPDATE_FREQUENCY + + def perform + return if Rails.configuration.x.email_domains_whitelist.present? || ENV['DISABLE_AUTOMATIC_SWITCHING_TO_APPROVED_REGISTRATIONS'] == 'true' + return unless Setting.registrations_mode == 'open' + + switch_to_approval_mode! unless active_moderators? + end + + private + + def active_moderators? + User.those_who_can(:manage_reports).exists?(current_sign_in_at: OPEN_REGISTRATIONS_MODERATOR_THRESHOLD.ago...) + end + + def switch_to_approval_mode! + Setting.registrations_mode = 'approved' + + User.those_who_can(:manage_settings).includes(:account).find_each do |user| + AdminMailer.auto_close_registrations(recipient: user.account).deliver_later + end + end +end diff --git a/config/locales/en.yml b/config/locales/en.yml index 14bbd2897e7462..23b534a85a9550 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -749,6 +749,7 @@ en: disabled: To no one users: To logged-in local users registrations: + moderation_recommandation: Please make sure you have an adequate and reactive moderation team before you open registrations to everyone! preamble: Control who can create an account on your server. title: Registrations registrations_mode: @@ -909,6 +910,9 @@ en: title: Webhooks webhook: Webhook admin_mailer: + auto_close_registrations: + body: Due to a lack of recent moderator activity, registrations on %{instance} have been automatically switched to requiring manual review, to prevent %{instance} from being used as a platform for potential bad actors. You can switch it back to open registrations at any time. + subject: Registrations for %{instance} have been automatically switched to requiring approval new_appeal: actions: delete_statuses: to delete their posts diff --git a/config/settings.yml b/config/settings.yml index f0b09dd5c88700..fa63fb05336c52 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -9,7 +9,7 @@ defaults: &defaults site_terms: '' site_contact_username: '' site_contact_email: '' - registrations_mode: 'open' + registrations_mode: 'none' profile_directory: true closed_registrations_message: '' open_deletion: true diff --git a/config/sidekiq.yml b/config/sidekiq.yml index be6861eaed70c2..19b71eed345ef2 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -62,3 +62,7 @@ interval: 1 minute class: Scheduler::SuspendedUserCleanupScheduler queue: scheduler + auto_close_registrations_scheduler: + interval: 1 hour + class: Scheduler::AutoCloseRegistrationsScheduler + queue: scheduler diff --git a/docker-compose.yml b/docker-compose.yml index 3e4a14413ab1ec..acaeafa633576f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -56,7 +56,7 @@ services: web: build: . - image: ghcr.io/mastodon/mastodon:v4.1.15 + image: ghcr.io/mastodon/mastodon:v4.1.16 restart: always env_file: .env.production command: bash -c "rm -f /mastodon/tmp/pids/server.pid; bundle exec rails s -p 3000" @@ -77,7 +77,7 @@ services: streaming: build: . - image: ghcr.io/mastodon/mastodon:v4.1.15 + image: ghcr.io/mastodon/mastodon:v4.1.16 restart: always env_file: .env.production command: node ./streaming @@ -95,7 +95,7 @@ services: sidekiq: build: . - image: ghcr.io/mastodon/mastodon:v4.1.15 + image: ghcr.io/mastodon/mastodon:v4.1.16 restart: always env_file: .env.production command: bundle exec sidekiq diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index 29a22eff3b8ea6..d5babbe1d8d35e 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -13,7 +13,7 @@ def minor end def patch - 15 + 16 end def flags diff --git a/spec/models/setting_spec.rb b/spec/models/setting_spec.rb index 3ccc21d6c41bed..ef81b97c9cac36 100644 --- a/spec/models/setting_spec.rb +++ b/spec/models/setting_spec.rb @@ -142,22 +142,12 @@ context 'records includes nothing' do let(:records) { [] } - context 'default_value is not a Hash' do - it 'includes Setting with value of default_value' do - setting = described_class.all_as_records[key] - - expect(setting).to be_kind_of Setting - expect(setting).to have_attributes(var: key) - expect(setting).to have_attributes(value: 'default_value') - end - end - - context 'default_value is a Hash' do - let(:default_value) { { 'foo' => 'fuga' } } + it 'includes Setting with value of default_value' do + setting = described_class.all_as_records[key] - it 'returns {}' do - expect(described_class.all_as_records).to eq({}) - end + expect(setting).to be_a described_class + expect(setting).to have_attributes(var: key) + expect(setting).to have_attributes(value: default_value) end end end diff --git a/spec/services/activitypub/process_account_service_spec.rb b/spec/services/activitypub/process_account_service_spec.rb index 2b20d17b1bc984..fbb5d58978bcec 100644 --- a/spec/services/activitypub/process_account_service_spec.rb +++ b/spec/services/activitypub/process_account_service_spec.rb @@ -3,7 +3,7 @@ RSpec.describe ActivityPub::ProcessAccountService, type: :service do subject { described_class.new } - context 'property values' do + context 'with property values, an avatar, and a profile header' do let(:payload) do { id: 'https://foo.test', @@ -14,19 +14,50 @@ { type: 'PropertyValue', name: 'Occupation', value: 'Unit test' }, { type: 'PropertyValue', name: 'non-string', value: ['foo', 'bar'] }, ], + image: { + type: 'Image', + mediaType: 'image/png', + url: 'https://foo.test/image.png', + }, + icon: { + type: 'Image', + url: [ + { + mediaType: 'image/png', + href: 'https://foo.test/icon.png', + }, + ], + }, }.with_indifferent_access end - it 'parses out of attachment' do + before do + stub_request(:get, 'https://foo.test/image.png').to_return(request_fixture('avatar.txt')) + stub_request(:get, 'https://foo.test/icon.png').to_return(request_fixture('avatar.txt')) + end + + it 'parses property values, avatar and profile header as expected' do account = subject.call('alice', 'example.com', payload) - expect(account.fields).to be_a Array - expect(account.fields.size).to eq 2 - expect(account.fields[0]).to be_a Account::Field - expect(account.fields[0].name).to eq 'Pronouns' - expect(account.fields[0].value).to eq 'They/them' - expect(account.fields[1]).to be_a Account::Field - expect(account.fields[1].name).to eq 'Occupation' - expect(account.fields[1].value).to eq 'Unit test' + + expect(account.fields) + .to be_an(Array) + .and have_attributes(size: 2) + expect(account.fields.first) + .to be_an(Account::Field) + .and have_attributes( + name: eq('Pronouns'), + value: eq('They/them') + ) + expect(account.fields.last) + .to be_an(Account::Field) + .and have_attributes( + name: eq('Occupation'), + value: eq('Unit test') + ) + expect(account).to have_attributes( + avatar_remote_url: 'https://foo.test/icon.png', + header_remote_url: 'https://foo.test/image.png' + ) end end diff --git a/spec/services/verify_link_service_spec.rb b/spec/services/verify_link_service_spec.rb index 8f65f3a8462b51..0d08ec14eb5e57 100644 --- a/spec/services/verify_link_service_spec.rb +++ b/spec/services/verify_link_service_spec.rb @@ -78,24 +78,25 @@ " - " end - it 'marks the field as not verified' do - expect(field.verified?).to be false + it 'marks the field as verified' do + expect(field.verified?).to be true end end - context 'when a link back might be truncated' do + context 'when a link tag might be truncated' do let(:html) do " -