Skip to content

Commit

Permalink
Sync via git repository
Browse files Browse the repository at this point in the history 
This patch allows Watson to sync via a simple git repository, making it
unnecessary to run a specific backend. Instead, any git repository with
write access will do.

You can configure a git backend by setting something like:

    watson config backend.repo [email protected]:user/repo.git

If y repository is set, the `sync` command will try using git. If not,
the old backend server is used if `server` and `token` are set.
  • Loading branch information
@lkiesow
lkiesow committed Feb 6, 2023
1 parent d9de4fc commit f10961a
Show file tree
Hide file tree
Showing 4 changed files with 129 additions and 45 deletions.
3 changes: 3 additions & 0 deletions docs/user-guide/commands.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -739,6 +739,9 @@ Example:

$ watson config backend.url http://localhost:4242
$ watson config backend.token 7e329263e329
or
$ watson config backend.repo [email protected]:user/repo.git

$ watson sync
Received 42 frames from the server
Pushed 23 frames to the server
Expand Down
19 changes: 16 additions & 3 deletions docs/user-guide/configuration.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,21 @@ $ watson config -e

### Backend

At this time there is no official backend for Watson. We are working on it. But in a near future, you will be able to synchronize Watson with a public (or your private) repository via the [`sync`](./commands.md#sync) command. To configure your repository please set up the `[backend]` section.
You will be able to synchronize Watson with a public (or your private) repository via the [`sync`](./commands.md#sync) command.
To configure your repository please set up the `[backend]` section.
You have two options for synchronization:

- The [crick](https://github.com/TailorDev/crick) server
- A git repository

If using crick, set `backend.url` and `backend.token`.
If using a git repository, set `backend.repo`.


#### `backend.repo` (default: empty)

The remote URL of a git repository to clone.
Something like `[email protected]:user/repo.git`.

#### `backend.url` (default: empty)

Expand Down Expand Up @@ -225,8 +239,7 @@ A basic configuration file looks like the following:
# Watson configuration

[backend]
url = https://api.crick.fr
token = yourapitoken
repo = [email protected]:user/repo.git

[options]
stop_on_start = true
Expand Down
7 changes: 5 additions & 2 deletions watson/cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1519,16 +1519,19 @@ def sync(watson):
\b
$ watson config backend.url http://localhost:4242
$ watson config backend.token 7e329263e329
or
$ watson config backend.repo [email protected]:user/repo.git
\b
$ watson sync
Received 42 frames from the server
Pushed 23 frames to the server
"""
last_pull = arrow.utcnow()
pulled = watson.pull()
click.echo("Received {} frames from the server".format(len(pulled)))
click.echo("Received {} frames from the server".format(pulled))

pushed = watson.push(last_pull)
click.echo("Pushed {} frames to the server".format(len(pushed)))
click.echo("Pushed {} frames to the server".format(pushed))

watson.last_sync = arrow.utcnow()
watson.save()
Expand Down
145 changes: 105 additions & 40 deletions watson/watson.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from configparser import Error as CFGParserError
import arrow
import click
import subprocess

from .config import ConfigParser
from .frames import Frames
Expand Down Expand Up @@ -54,6 +55,7 @@ def __init__(self, **kwargs):
self.frames_file = os.path.join(self._dir, 'frames')
self.state_file = os.path.join(self._dir, 'state')
self.last_sync_file = os.path.join(self._dir, 'last_sync')
self.sync_dir = os.path.join(self._dir, 'sync_repo')

if 'frames' in kwargs:
self.frames = kwargs['frames']
Expand Down Expand Up @@ -371,43 +373,63 @@ def _get_remote_projects(self):
return self._remote_projects['projects']

def pull(self):
import requests
dest, headers = self._get_request_info('frames')

try:
response = requests.get(
dest, params={'last_sync': self.last_sync}, headers=headers
)
assert response.status_code == 200
except requests.ConnectionError:
raise WatsonError("Unable to reach the server.")
except AssertionError:
raise WatsonError(
"An error occurred with the remote "
"server: {}".format(response.json())
)
repo = self.config.get('backend', 'repo')
if repo:
# clone git repository if necessary
if not os.path.isdir(self.sync_dir):
sync_dir = self.sync_dir
subprocess.run(['git', 'clone', repo, sync_dir], check=True)

# git pull
subprocess.run(['git', 'pull'], cwd=self.sync_dir, check=True)
sync_file = os.path.join(self.sync_dir, 'frames')
try:
with open(sync_file, 'r') as f:
frames = json.load(f)
except FileNotFoundError:
frames = []
else:
import requests
dest, headers = self._get_request_info('frames')

try:
response = requests.get(
dest, params={'last_sync': self.last_sync}, headers=headers
)
assert response.status_code == 200
except requests.ConnectionError:
raise WatsonError("Unable to reach the server.")
except AssertionError:
raise WatsonError(
"An error occurred with the remote "
"server: {}".format(response.json())
)

frames = response.json() or ()
frames = response.json() or ()

updated_frames = 0
for frame in frames:
frame_id = uuid.UUID(frame['id']).hex
self.frames[frame_id] = (
frame['project'],
frame['begin_at'],
frame['end_at'],
frame['tags']
)
try:
self.frames[frame_id]
except KeyError:
updated_frames += 1
self.frames[frame_id] = (
frame['project'],
frame['begin_at'],
frame['end_at'],
frame['tags']
)

return frames
return updated_frames

def push(self, last_pull):
import requests
dest, headers = self._get_request_info('frames/bulk')

frames = []

for frame in self.frames:
if last_pull > frame.updated_at > self.last_sync:
repo = self.config.get('backend', 'repo')
if repo:
for frame in self.frames:
frames.append({
'id': uuid.UUID(frame.id).urn,
'begin_at': str(frame.start.to('utc')),
Expand All @@ -416,21 +438,64 @@ def push(self, last_pull):
'tags': frame.tags
})

try:
response = requests.post(dest, json.dumps(frames), headers=headers)
assert response.status_code == 201
except requests.ConnectionError:
raise WatsonError("Unable to reach the server.")
except AssertionError:
raise WatsonError(
"An error occurred with the remote server (status: {}). "
"Response was:\n{}".format(
response.status_code,
response.text
# Get number of synced frames
sync_file = os.path.join(self.sync_dir, 'frames')
try:
with open(sync_file, 'r') as f:
n_frames = len(json.load(f))
except FileNotFoundError:
n_frames = 0

# Write frames to repo
with open(sync_file, 'w') as f:
json.dump(frames, f, indent=2)

# Check if anything has changed
try:
command = ['git', 'diff', '--quiet']
subprocess.run(command, cwd=self.sync_dir, check=True)
return 0
except subprocess.CalledProcessError:
pass

# git push
cwd = self.sync_dir
subprocess.run(['git', 'add', 'frames'], cwd=cwd, check=True)
subprocess.run(['git', 'commit', '-m', '...'], cwd=cwd, check=True)
subprocess.run(['git', 'push'], cwd=cwd, check=True)

return len(frames) - n_frames

else:
import requests
dest, headers = self._get_request_info('frames/bulk')

for frame in self.frames:
if last_pull > frame.updated_at > self.last_sync:
frames.append({
'id': uuid.UUID(frame.id).urn,
'begin_at': str(frame.start.to('utc')),
'end_at': str(frame.stop.to('utc')),
'project': frame.project,
'tags': frame.tags
})

try:
body = json.dumps(frames)
response = requests.post(dest, body, headers=headers)
assert response.status_code == 201
except requests.ConnectionError:
raise WatsonError("Unable to reach the server.")
except AssertionError:
raise WatsonError(
"An error occurred with the remote server (status: {}). "
"Response was:\n{}".format(
response.status_code,
response.text
)
)
)

return frames
return len(frames)

def merge_report(self, frames_with_conflict):
conflict_file_frames = Frames(self._load_json_file(
Expand Down

0 comments on commit f10961a

Please sign in to comment.