initial commit

core/.bundle/config

@@ -0,0 +1,2 @@
+---
+BUNDLE_PATH: "vendor/bundle"

core/.env.sample

@@ -0,0 +1,3 @@
+ADDRESS="[email protected]"
+PASSWORD="your imap password"
+DATABASE="db.sqlite3"

core/.gitignore

@@ -0,0 +1,3 @@
+.env
+*.sqlite3
+vendor/

core/.ruby-version

@@ -0,0 +1 @@
+3.1.2

core/Gemfile

@@ -0,0 +1,10 @@
+source "https://rubygems.org"
+
+gem "mail"
+gem 'net-smtp', require: false
+gem 'net-imap', require: false
+gem 'net-pop', require: false
+gem "sequel"
+gem "debug"
+gem "sqlite3"
+gem "dotenv"

core/Gemfile.lock

@@ -0,0 +1,42 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    debug (1.6.2)
+      irb (>= 1.3.6)
+      reline (>= 0.3.1)
+    dotenv (2.8.1)
+    io-console (0.5.11)
+    irb (1.4.1)
+      reline (>= 0.3.0)
+    mail (2.7.1)
+      mini_mime (>= 0.1.1)
+    mini_mime (1.1.2)
+    net-imap (0.3.1)
+      net-protocol
+    net-pop (0.1.2)
+      net-protocol
+    net-protocol (0.1.3)
+      timeout
+    net-smtp (0.3.2)
+      net-protocol
+    reline (0.3.1)
+      io-console (~> 0.5)
+    sequel (5.60.1)
+    sqlite3 (1.5.0-x86_64-linux)
+    timeout (0.3.0)
+
+PLATFORMS
+  x86_64-linux
+
+DEPENDENCIES
+  debug
+  dotenv
+  mail
+  net-imap
+  net-pop
+  net-smtp
+  sequel
+  sqlite3
+
+BUNDLED WITH
+   2.3.21

core/README.md

@@ -0,0 +1,37 @@
+# NittyMail Core
+
+This folder contains some common functionality, among which is a simple syncing script that will download all messages in a Gmail account to an sqlite3 database.
+
+## Usage
+
+You will need to enable IMAP for the Gmail account that will be synced. Support documentation can be found at <https://support.google.com/mail/answer/7126229>
+
+If the account has 2FA enabled, an app password will need to be generated and used instead of the account password. More documentation found at <https://support.google.com/accounts/answer/185833>
+
+Configuration is done through the `.env` file; there is a sample `.env.sample` that can be copied and modified as necessary.
+
+A docker-compose.yml has been provided for convenience. With `docker ` and `docker-compose` installed:
+
+``` bash
+DOCKER_UID="$(id -u)" GID="$(id -g)" docker-compose run --rm ruby bundle
+DOCKER_UID="$(id -u)" GID="$(id -g)" docker-compose run --rm ruby ./sync.rb
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at <https://github.com/parasquid/nittymail/issues>
+
+## License
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.

core/docker-compose.yml

@@ -0,0 +1,11 @@
+version: '3.2'
+services:
+  ruby:
+    platform: linux/amd64
+    image: ruby:3.1.2
+    user: "${DOCKER_UID}:${GID}"
+    volumes:
+      - ./:/app
+      - ./.bundle:/usr/local/bundle
+    working_dir: /app
+    command: "bin/console"

core/sync.rb

@@ -0,0 +1,164 @@
+#!/usr/bin/env ruby
+
+# Copyright 2022 parasquid
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+require 'bundler/setup'
+require 'dotenv/load'
+require "debug"
+require "mail"
+require "sequel"
+
+# patch only this instance of Net::IMAP::ResponseParser
+def patch(gmail_imap)
+  class << gmail_imap.instance_variable_get("@parser")
+
+    # copied from https://github.com/ruby/net-imap/blob/master/lib/net/imap/response_parser.rb#L193
+    def msg_att(n)
+      match(T_LPAR)
+      attr = {}
+      while true
+        token = lookahead
+        case token.symbol
+        when T_RPAR
+          shift_token
+          break
+        when T_SPACE
+          shift_token
+          next
+        end
+        case token.value
+        when /\A(?:ENVELOPE)\z/ni
+          name, val = envelope_data
+        when /\A(?:FLAGS)\z/ni
+          name, val = flags_data
+        when /\A(?:INTERNALDATE)\z/ni
+          name, val = internaldate_data
+        when /\A(?:RFC822(?:\.HEADER|\.TEXT)?)\z/ni
+          name, val = rfc822_text
+        when /\A(?:RFC822\.SIZE)\z/ni
+          name, val = rfc822_size
+        when /\A(?:BODY(?:STRUCTURE)?)\z/ni
+          name, val = body_data
+        when /\A(?:UID)\z/ni
+          name, val = uid_data
+        when /\A(?:MODSEQ)\z/ni
+          name, val = modseq_data
+
+        # adding in Gmail extended attributes
+        # see https://gist.github.com/kellyredding/2712611
+        when /\A(?:X-GM-LABELS)\z/ni
+          name, val = flags_data
+        when /\A(?:X-GM-MSGID)\z/ni
+          name, val = uid_data
+        when /\A(?:X-GM-THRID)\z/ni
+          name, val = uid_data
+
+        else
+          parse_error("unknown attribute `%s' for {%d}", token.value, n)
+        end
+        attr[name] = val
+      end
+      return attr
+    end
+
+  end
+  gmail_imap
+end
+
+# IMAP
+imap_address = ENV["ADDRESS"]
+imap_password = ENV["PASSWORD"]
+Mail.defaults do
+  retriever_method :imap, :address    => "imap.gmail.com",
+                          :port       => 993,
+                          :user_name  => imap_address,
+                          :password   => imap_password,
+                          :enable_ssl => true
+end
+
+DB = Sequel.sqlite(ENV["DATABASE"])
+
+unless DB.table_exists?(:email)
+  DB.create_table :email do
+    primary_key :id
+    String :address, index: true
+    String :mailbox, index: true
+    Bignum :uid, index: true, default: 0
+    Integer :uidvalidity, index: true, default: 0
+
+    String :message_id, index: true
+    DateTime :date, index: true
+    String :from, index: true
+    String :subject
+
+    Boolean :has_attachments, index: true, default: false
+    String :x_gm_msgid
+    String :encoded
+
+    unique [:mailbox, :uid, :uidvalidity]
+    index [:mailbox, :uidvalidity]
+  end
+end
+
+email = DB[:email]
+
+# get all mailboxes
+mailboxes = Mail.connection { |imap| imap.list '', '*' }
+mailboxes.each do |mailbox|
+
+  # mailboxes with attr :Noselect cannpt be selected so we skip those
+  next if mailbox.attr.include?(:Noselect)
+
+  mbox_name = mailbox.name
+  puts "processing mailbox #{mbox_name}"
+
+  # get the max uid for a mailbox, paying attention to the uidvalidity
+  # we have to "throw away" the cached records if the uidvalidity changes
+  uidvalidity = 1
+  max_uid = 1
+  Mail.connection do |imap|
+    imap.select(mbox_name)
+    uidvalidity =  imap.responses["UIDVALIDITY"]&.first || 1
+    max_uid = email.where(mailbox: mbox_name, uidvalidity: uidvalidity).max(:uid) || 1
+  end
+
+  Mail.find read_only: true, count: :all, mailbox: mbox_name, keys: "#{max_uid}:*" do |mail, imap, uid|
+
+    # patch in Gmail specific extesnions
+    patch(imap)
+    x_gm_msgid = imap.uid_fetch(uid, ['X-GM-MSGID']).first.attr["X-GM-MSGID"].to_s
+
+    begin
+      email.insert(
+        address: imap_address,
+        mailbox: mbox_name.force_encoding("UTF-8"),
+        uid: uid,
+        uidvalidity: uidvalidity,
+
+        message_id: mail.message_id.force_encoding("UTF-8"),
+        date: mail.date,
+        from: mail.from.join(", ").force_encoding("UTF-8"),
+        subject: mail.subject.force_encoding("UTF-8"),
+        has_attachments: mail.has_attachments?,
+
+        x_gm_msgid: x_gm_msgid.force_encoding("UTF-8"),
+        encoded: mail.encoded.force_encoding("UTF-8")
+      )
+    rescue Sequel::UniqueConstraintViolation
+      puts "#{mbox_name} #{uid} #{uidvalidity} already exists, skipping ..."
+    end
+  end
+end