Skip to content

Commit

Permalink
Merge branch 'main' into new-new-copier
Browse files Browse the repository at this point in the history
  • Loading branch information
chrisvittal committed Aug 5, 2024
2 parents 9c25bc0 + 84dd26c commit 1a33173
Show file tree
Hide file tree
Showing 80 changed files with 1,788 additions and 697 deletions.
19 changes: 19 additions & 0 deletions .github/workflows/require-linked-issue.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
name: require linked issue

on:
pull_request_target:
branches: ["main"]
types: [opened, edited, reopened, synchronize]

jobs:
check_pull_requests:
runs-on: ubuntu-latest
name: require linked issue
permissions:
issues: read
pull-requests: read
steps:
- uses: nearform-actions/github-action-check-linked-issues@v1
id: check-linked-issues
with:
comment: false
24 changes: 2 additions & 22 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -99,36 +99,16 @@ install-dev-requirements:
-r batch/pinned-requirements.txt \
-r ci/pinned-requirements.txt

hail/python/hailtop/pinned-requirements.txt: hail/python/hailtop/requirements.txt
.PHONY: generate-pip-lockfiles
generate-pip-lockfiles:
./generate-pip-lockfile.sh hail/python/hailtop

hail/python/pinned-requirements.txt: hail/python/hailtop/pinned-requirements.txt hail/python/requirements.txt
./generate-pip-lockfile.sh hail/python

hail/python/dev/pinned-requirements.txt: hail/python/pinned-requirements.txt hail/python/dev/requirements.txt
./generate-pip-lockfile.sh hail/python/dev

gear/pinned-requirements.txt: hail/python/pinned-requirements.txt hail/python/dev/pinned-requirements.txt hail/python/hailtop/pinned-requirements.txt gear/requirements.txt
./generate-pip-lockfile.sh gear

web_common/pinned-requirements.txt: gear/pinned-requirements.txt web_common/requirements.txt
./generate-pip-lockfile.sh web_common

batch/pinned-requirements.txt: web_common/pinned-requirements.txt batch/requirements.txt
./generate-pip-lockfile.sh batch

ci/pinned-requirements.txt: web_common/pinned-requirements.txt ci/requirements.txt
./generate-pip-lockfile.sh ci

.PHONY: generate-pip-lockfiles
generate-pip-lockfiles: hail/python/hailtop/pinned-requirements.txt
generate-pip-lockfiles: hail/python/pinned-requirements.txt
generate-pip-lockfiles: hail/python/dev/pinned-requirements.txt
generate-pip-lockfiles: gear/pinned-requirements.txt
generate-pip-lockfiles: web_common/pinned-requirements.txt
generate-pip-lockfiles: batch/pinned-requirements.txt
generate-pip-lockfiles: ci/pinned-requirements.txt

$(HAILTOP_VERSION):
$(MAKE) -C hail python/hailtop/hail_version

Expand Down
10 changes: 9 additions & 1 deletion auth/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
ARG BASE_IMAGE={{ hail_ubuntu_image.image }}
FROM $BASE_IMAGE

RUN curl -sLO https://github.com/tailwindlabs/tailwindcss/releases/download/v3.4.3/tailwindcss-linux-x64 && \
chmod +x tailwindcss-linux-x64 && \
mv tailwindcss-linux-x64 /usr/bin/tailwindcss

COPY hail/python/hailtop/pinned-requirements.txt hailtop-requirements.txt
COPY gear/pinned-requirements.txt gear-requirements.txt
COPY web_common/pinned-requirements.txt web_common-requirements.txt
Expand All @@ -16,12 +20,16 @@ COPY hail/python/hailtop /hailtop/hailtop/
COPY gear/setup.py /gear/setup.py
COPY gear/gear /gear/gear/

COPY web_common/setup.py web_common/MANIFEST.in /web_common/
COPY web_common/setup.py web_common/MANIFEST.in web_common/input.css web_common/tailwind.config.js /web_common/
COPY web_common/web_common /web_common/web_common/

COPY auth/setup.py auth/MANIFEST.in /auth/
COPY auth/auth /auth/auth/

RUN cd web_common && \
mkdir web_common/static/css && \
tailwindcss -i input.css -o web_common/static/css/output.css

RUN hail-pip-install /hailtop /gear /web_common /auth && \
rm -rf /hailtop /gear /web_common /auth

Expand Down
16 changes: 8 additions & 8 deletions auth/auth/templates/index.html
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
{% extends "layout.html" %}
{% block title %}Home{% endblock %}
{% block head %}
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/animate.css/3.7.0/animate.min.css">
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/animate.css/3.7.0/animate.min.css">
{% endblock %}
{% block content %}
<div class="vcentered">
<h1 class="animated fadeInUp faster">
<a href="https://hail.is/" target="_blank">
Hail
</a>
</h1>
</div>
<div class="vcentered">
<h1 class="animated fadeInUp faster hover:cursor-pointer hover:underline text-5xl text-sky-700 underline-offset-4">
<a href="https://hail.is/" target="_blank">
Hail
</a>
</h1>
</div>
{% endblock %}
58 changes: 33 additions & 25 deletions auth/auth/templates/roles.html
Original file line number Diff line number Diff line change
@@ -1,30 +1,38 @@
{% from "utils.html" import submit_button %}
{% extends "layout.html" %}
{% block title %}Roles{% endblock %}
{% block content %}
<h1>Create Role</h1>
<form action="{{ base_path }}/roles" method="POST">
<div>Name: <input required name="name" /></div>
<input type="hidden" name="_csrf" value="{{ csrf_token }}"/>
<button>
Create
</button>
</form>
<h1 class='text-2xl font-light mb-4'>Create Role</h1>
<form action="{{ base_path }}/roles" method="POST">
<div class='w-64 flex-col space-y-2'>
<div class='flex justify-between'>
<span>Name</span>
<input class='border rounded-sm p-1 text-sm' required name="name" />
</div>
<input type="hidden" name="_csrf" value="{{ csrf_token }}" />
<div class='flex justify-end'>
{{ submit_button('Create') }}
</div>
</div>
</form>

<h1>Roles</h1>
<table class="data-table">
<thead>
<tr>
<th>ID</th>
<th>Name</th>
</tr>
</thead>
<tbody>
{% for role in roles %}
<tr>
<td class="numeric-cell">{{ role['id'] }}</td>
<td>{{ role['name'] }}</td>
</tr>
{% endfor %}
</tbody>
</table>
<br />

<h1 class='text-2xl font-light mb-4'>Roles</h1>
<table class='table-auto w-64'>
<thead>
<tr>
<th class='h-12 bg-slate-200 text-center font-light rounded-tl pl-2'>ID</th>
<th class='h-12 bg-slate-200 text-center font-light rounded-tr pr-2'>Name</th>
</tr>
</thead>
<tbody class='border border-collapse border-slate-50'>
{% for role in roles %}
<tr>
<td class='text-center'>{{ role['id'] }}</td>
<td class='text-center'>{{ role['name'] }}</td>
</tr>
{% endfor %}
</tbody>
</table>
{% endblock %}
43 changes: 22 additions & 21 deletions auth/auth/templates/user.html
Original file line number Diff line number Diff line change
@@ -1,25 +1,26 @@
{% from "utils.html" import submit_button %}
{% extends "layout.html" %}
{% block title %}User{% endblock %}
{% block content %}
<div id="profile" class="vcentered">
<h1>{{ userdata['username'] }}</h1>
{% if cloud == "gcp" %}
<p><b>Google Service Account: </b>{{ userdata['display_name'] }}</p>
{% endif %}
{% if cloud == "azure" %}
<p><b>Azure Service Principal Display Name: </b>{{ userdata['display_name'] }}</p>
<p><b>Azure Service Principal Application ID: </b>{{ userdata['hail_identity'] }}</p>
{% endif %}
<p><b>Trial Billing Project: </b>{{ userdata['trial_bp_name'] }}</p>
<form action="{{ auth_base_url }}/copy-paste-token" method="post">
<input type="hidden" name="_csrf" value="{{ csrf_token }}"/>
<button class="nb-create">Get a copy-paste login token</button>
</form>
<p>
<b>Notice:</b> The Hail system records your email address and IP address. Your email address
is recorded so that we can authenticate you. Your IP address is tracked as part of our
surveillance of all traffic to and from the Hail system. This broad surveillance enables the
protection of the Hail system from malicious actors.
</p>
</div>
<div id="profile" class="vcentered space-y-2">
<h1 class='text-4xl mb-4'>{{ userdata['username'] }}</h1>
{% if cloud == "gcp" %}
<p><b>Google Service Account: </b>{{ userdata['display_name'] }}</p>
{% endif %}
{% if cloud == "azure" %}
<p><b>Azure Service Principal Display Name: </b>{{ userdata['display_name'] }}</p>
<p><b>Azure Service Principal Application ID: </b>{{ userdata['hail_identity'] }}</p>
{% endif %}
<p><b>Trial Billing Project: </b>{{ userdata['trial_bp_name'] }}</p>
<form action="{{ auth_base_url }}/copy-paste-token" method="post">
<input type="hidden" name="_csrf" value="{{ csrf_token }}" />
{{ submit_button('Get a copy-paste login token') }}
</form>
<p>
<b>Notice:</b> The Hail system records your email address and IP address. Your email address
is recorded so that we can authenticate you. Your IP address is tracked as part of our
surveillance of all traffic to and from the Hail system. This broad surveillance enables the
protection of the Hail system from malicious actors.
</p>
</div>
{% endblock %}
128 changes: 80 additions & 48 deletions auth/auth/templates/users.html
Original file line number Diff line number Diff line change
@@ -1,53 +1,85 @@
{% from "utils.html" import submit_button, danger_button, success_check, fail_cross %}
{% extends "layout.html" %}
{% block title %}Users{% endblock %}
{% block content %}
<h1>Create User</h1>
<form action="{{ base_path }}/users" method="POST">
<div>Username: <input required name="username" /></div>
<div>Login ID: <input name="login_id" /></div>
<div><input type="checkbox" name="is_developer" value="1" /> Developer</div>
<div><input type="checkbox" name="is_service_account" value="1" /> Service Account</div>
<input type="hidden" name="_csrf" value="{{ csrf_token }}"/>
<button>
Create
</button>
</form>

<h1>Delete User</h1>
<form action="{{ base_path }}/users/delete" method="POST">
<div>User ID: <input name="id" /></div>
<div>Username: <input name="username" /></div>
<input type="hidden" name="_csrf" value="{{ csrf_token }}"/>
<button class="dangerous">
Delete
</button>
</form>
{% macro check_or_cross(b) %}
{% if b %}
{{ success_check() }}
{% else %}
{{ fail_cross() }}
{% endif %}
{% endmacro %}

{% block content %}
<div class='w-full md:w-2/5 flex flex-col md:flex-row md:space-x-16 justify-between'>
<div>
<h1 class='text-2xl font-light mb-4'>Create User</h1>
<form action="{{ base_path }}/users" method="POST">
<div class='flex-col space-y-2'>
<div class='flex justify-between space-x-4'>
<span>Username</span>
<input class='border rounded-sm p-1 text-sm' required name="username" />
</div>
<div class='flex justify-between space-x-4'>
<span>Login ID</span>
<input class='border rounded-sm p-1 text-sm' name="login_id" />
</div>
<div><input type="checkbox" name="is_developer" value="1" /> Developer</div>
<div><input type="checkbox" name="is_service_account" value="1" /> Service Account</div>
<input type="hidden" name="_csrf" value="{{ csrf_token }}" />
<div class='flex justify-end'>
{{ submit_button('Create') }}
</div>
</div>
</form>
</div>
<div>
<h1 class='text-2xl font-light mb-4'>Delete User</h1>
<form action="{{ base_path }}/users/delete" method="POST">
<div class='flex-col space-y-2'>
<div class='flex justify-between space-x-4'>
<span>User ID</span>
<input class='border rounded-sm p-1 text-sm' required name="id" />
</div>
<div class='flex justify-between space-x-4'>
<span>Username</span>
<input class='border rounded-sm p-1 text-sm' required name="username" />
</div>
<input type="hidden" name="_csrf" value="{{ csrf_token }}" />
<div class='flex justify-end'>
{{ danger_button('Delete') }}
</div>
</div>
</form>
</div>
</div>

<h1>Users</h1>
<table class="data-table">
<thead>
<tr>
<th>ID</th>
<th>Username</th>
<th>Login ID</th>
<th>Hail Identity</th>
<th>State</th>
<th>Developer</th>
<th>Robot</th>
</tr>
</thead>
<tbody>
{% for user in users %}
<tr>
<td class="numeric-cell">{{ user['id'] }}</td>
<td>{{ user['username'] }}</td>
<td>{{ user['login_id'] }}</td>
<td>{{ user['hail_identity'] }}</td>
<td>{{ user['state'] }}</td>
<td class="numeric-cell">{{ user['is_developer'] }}</td>
<td class="numeric-cell">{{ user['is_service_account'] }}</td>
</tr>
{% endfor %}
</tbody>
</table>
<br />
<h1 class='text-2xl font-light mb-4'>Users</h1>
<table class="table-auto w-full">
<thead>
<tr>
<th class='h-12 bg-slate-200 font-light rounded-tl pl-2'>ID</th>
<th class='h-12 bg-slate-200 font-light text-left'>Username</th>
<th class='h-12 bg-slate-200 font-light text-left'>Login ID</th>
<th class='h-12 bg-slate-200 font-light text-left'>Hail Identity</th>
<th class='h-12 bg-slate-200 font-light text-left'>State</th>
<th class='h-12 bg-slate-200 font-light'>Developer</th>
<th class='h-12 bg-slate-200 font-light rounded-tr pr-2'>Robot</th>
</tr>
</thead>
<tbody class='border border-collapse border-slate-50'>
{% for user in users %}
<tr class='border border-collapse hover:bg-slate-100 font-light'>
<td class='text-center'>{{ user['id'] }}</td>
<td>{{ user['username'] }}</td>
<td>{{ user['login_id'] }}</td>
<td>{{ user['hail_identity'] }}</td>
<td>{{ user['state'] }}</td>
<td class='text-center'>{{ check_or_cross(user['is_developer']) }}</td>
<td class='text-center'>{{ check_or_cross(user['is_service_account']) }}</td>
</tr>
{% endfor %}
</tbody>
</table>
{% endblock %}
12 changes: 0 additions & 12 deletions batch/Dockerfile.driver-nginx

This file was deleted.

4 changes: 2 additions & 2 deletions batch/batch/cloud/gcp/driver/disks.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,12 @@ async def delete_orphaned_disks(
params = {'filter': f'(labels.namespace = {namespace})'}

for zone in zones:
async for disk in await compute_client.list(f'/zones/{zone}/disks', params=params):
async for disk in compute_client.list(f'/zones/{zone}/disks', params=params):
disk_name = disk['name']
instance_name = disk['labels']['instance-name']
instance = inst_coll_manager.get_instance(instance_name)

creation_timestamp_msecs = parse_timestamp_msecs(disk.get('creationTimestamp'))
creation_timestamp_msecs = parse_timestamp_msecs(disk['creationTimestamp'])
last_attach_timestamp_msecs = parse_timestamp_msecs(disk.get('lastAttachTimestamp'))
last_detach_timestamp_msecs = parse_timestamp_msecs(disk.get('lastDetachTimestamp'))

Expand Down
2 changes: 1 addition & 1 deletion batch/batch/cloud/gcp/driver/zones.py
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ async def fetch_machine_valid_zones(
zones = [url_basename(z) for r in region_info.values() for z in r['zones']]
machine_family_valid_zones = collections.defaultdict(set)
for zone in zones:
async for machine_type in await compute_client.list(f'/zones/{zone}/machineTypes'):
async for machine_type in compute_client.list(f'/zones/{zone}/machineTypes'):
machine_family = machine_type['name'].split('-')[0]
machine_family_valid_zones[machine_family].add(machine_type['zone'])
return machine_family_valid_zones
Expand Down
Loading

0 comments on commit 1a33173

Please sign in to comment.