Skip to content

Commit

Permalink
Remove metadata backfill task (#15526)
Browse files Browse the repository at this point in the history
  • Loading branch information
@di
di authored Mar 4, 2024
1 parent 26b9b38 commit 2e4117d
Show file tree
Hide file tree
Showing 3 changed files with 0 additions and 378 deletions.
272 changes: 0 additions & 272 deletions tests/unit/packaging/test_tasks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,33 +10,25 @@
# See the License for the specific language governing permissions and
# limitations under the License.

import builtins
import tempfile

from contextlib import contextmanager
from itertools import product
from pathlib import Path
from zipfile import BadZipFile

import pretend
import pytest

from google.cloud.bigquery import SchemaField
from pip._internal.exceptions import InvalidWheel, UnsupportedWheel
from wtforms import Field, Form, StringField

import warehouse.packaging.tasks

from warehouse.accounts.models import WebAuthn
from warehouse.metrics.interfaces import IMetricsService
from warehouse.packaging.interfaces import IFileStorage
from warehouse.packaging.models import Description
from warehouse.packaging.tasks import (
check_file_cache_tasks_outstanding,
compute_2fa_metrics,
compute_packaging_metrics,
metadata_backfill,
metadata_backfill_individual,
sync_bigquery_release_files,
sync_file_to_cache,
update_bigquery_release_files,
Expand Down Expand Up @@ -896,267 +888,3 @@ def test_compute_2fa_metrics(db_request, monkeypatch):
pretend.call("warehouse.2fa.total_users_with_webauthn_enabled", 1),
pretend.call("warehouse.2fa.total_users_with_two_factor_enabled", 2),
]


def test_metadata_backfill(db_request, monkeypatch, metrics):
project = ProjectFactory()
release1 = ReleaseFactory(project=project)
release2 = ReleaseFactory(project=project)
FileFactory(release=release1, packagetype="sdist")
FileFactory(
release=release1,
packagetype="bdist_wheel",
metadata_file_sha256_digest="d34db33f",
)
FileFactory(release=release2, packagetype="sdist")
backfillable_file = FileFactory(
release=release2, packagetype="bdist_wheel", metadata_file_sha256_digest=None
)

delay = pretend.call_recorder(lambda x: None)
db_request.task = pretend.call_recorder(lambda x: pretend.stub(delay=delay))
db_request.registry.settings["metadata_backfill.batch_size"] = 500

metadata_backfill(db_request)

assert db_request.task.calls == [pretend.call(metadata_backfill_individual)]
assert delay.calls == [pretend.call(backfillable_file.id)]

assert metrics.increment.calls == [
pretend.call("warehouse.packaging.metadata_backfill.tasks"),
]
assert metrics.gauge.calls == [
pretend.call("warehouse.packaging.metadata_backfill.remaining", 1)
]


@pytest.mark.parametrize(
"path, requested_path",
[
# Regular path
(
"8a/6a/19...3b/pip-24.0-py3-none-any.whl",
"8a/6a/19...3b/pip-24.0-py3-none-any.whl",
),
# Path with characters that need quoted
(
"da/5d/cc...0d/scalg-0.1.1#-py3-none-any.whl",
"da/5d/cc...0d/scalg-0.1.1%23-py3-none-any.whl",
),
],
)
def test_metadata_backfill_individual(
db_request, monkeypatch, metrics, path, requested_path
):
project = ProjectFactory()
release1 = ReleaseFactory(project=project)
release2 = ReleaseFactory(project=project)
FileFactory(release=release1, packagetype="sdist")
FileFactory(
release=release1,
packagetype="bdist_wheel",
metadata_file_sha256_digest="d34db33f",
)
FileFactory(release=release2, packagetype="sdist")
backfillable_file = FileFactory(
release=release2,
packagetype="bdist_wheel",
metadata_file_sha256_digest=None,
path=path,
)

metadata_contents = b"some\nmetadata\ncontents"
stub_dist = pretend.stub(
_dist=pretend.stub(_files={Path("METADATA"): metadata_contents})
)
stub_session = pretend.stub()
dist_from_wheel_url = pretend.call_recorder(
lambda project_name, file_url, session: stub_dist
)
monkeypatch.setattr(
warehouse.packaging.tasks, "dist_from_wheel_url", dist_from_wheel_url
)
monkeypatch.setattr(warehouse.packaging.tasks, "PipSession", lambda: stub_session)
archive_storage = pretend.stub(
store=pretend.call_recorder(lambda path_out, path_in, meta: None),
)
cache_storage = pretend.stub(
store=pretend.call_recorder(lambda path_out, path_in, meta: None),
)
db_request.find_service = pretend.call_recorder(
lambda iface, name=None, context=None: {
IFileStorage: {
"archive": archive_storage,
"cache": cache_storage,
},
IMetricsService: {None: metrics},
}[iface][name]
)

@contextmanager
def mock_temporary_directory():
yield "/tmp/wutang"

monkeypatch.setattr(tempfile, "TemporaryDirectory", mock_temporary_directory)

mock_write = pretend.call_recorder(lambda value: None)

@contextmanager
def mock_open(filename, perms):
yield pretend.stub(write=mock_write)

monkeypatch.setattr(builtins, "open", mock_open)

db_request.registry.settings["files.url"] = (
"https://files.example.com/packages/{path}"
)

metadata_backfill_individual(db_request, backfillable_file.id)

assert dist_from_wheel_url.calls == [
pretend.call(
project.normalized_name,
f"https://files.example.com/packages/{requested_path}",
stub_session,
)
]

assert backfillable_file.metadata_file_sha256_digest == (
"e85ce4c9e2d2eddba19c396ed04470efaa2a9c2a6b3c6463e6876a41e55d828d"
)
assert backfillable_file.metadata_file_blake2_256_digest == (
"39cc629504be4087d48889e8666392bd379b91e1826e269cd8467bb29298da82"
)
assert (
archive_storage.store.calls
== cache_storage.store.calls
== [
pretend.call(
backfillable_file.metadata_path,
f"/tmp/wutang/{backfillable_file.filename}.metadata",
meta={
"project": project.normalized_name,
"version": release2.version,
"package-type": backfillable_file.packagetype,
"python-version": backfillable_file.python_version,
},
),
]
)

assert metrics.increment.calls == [
pretend.call("warehouse.packaging.metadata_backfill.files"),
]


@pytest.mark.parametrize(
"exception", [InvalidWheel("foo", "bar"), UnsupportedWheel, BadZipFile, KeyError]
)
def test_metadata_backfill_file_invalid_wheel(
db_request, monkeypatch, metrics, exception
):
project = ProjectFactory()
release = ReleaseFactory(project=project)
backfillable_file = FileFactory(
release=release, packagetype="bdist_wheel", metadata_file_sha256_digest=None
)

stub_session = pretend.stub()
monkeypatch.setattr(warehouse.packaging.tasks, "PipSession", lambda: stub_session)
dist_from_wheel_url = pretend.raiser(exception)
monkeypatch.setattr(
warehouse.packaging.tasks, "dist_from_wheel_url", dist_from_wheel_url
)
db_request.find_service = pretend.call_recorder(
lambda iface, name=None, context=None: {
IFileStorage: {
"archive": pretend.stub(),
"cache": pretend.stub(),
},
IMetricsService: {None: metrics},
}[iface][name]
)
db_request.registry.settings["files.url"] = (
"https://files.example.com/packages/{path}"
)

assert backfillable_file.metadata_file_unbackfillable is False

metadata_backfill_individual(db_request, backfillable_file.id)

assert backfillable_file.metadata_file_unbackfillable is True
assert metrics.increment.calls == []


def test_metadata_backfill_file_oserror(db_request, monkeypatch, metrics):
project = ProjectFactory()
release = ReleaseFactory(project=project)
backfillable_file = FileFactory(
release=release, packagetype="bdist_wheel", metadata_file_sha256_digest=None
)

metadata_contents = b"some\nmetadata\ncontents"
stub_dist = pretend.stub(
_dist=pretend.stub(_files={Path("METADATA"): metadata_contents})
)
stub_session = pretend.stub()
dist_from_wheel_url = pretend.call_recorder(
lambda project_name, file_url, session: stub_dist
)
monkeypatch.setattr(
warehouse.packaging.tasks, "dist_from_wheel_url", dist_from_wheel_url
)
stub_session = pretend.stub()
monkeypatch.setattr(warehouse.packaging.tasks, "PipSession", lambda: stub_session)

@contextmanager
def mock_open(filename, perms):
raise OSError

monkeypatch.setattr(builtins, "open", mock_open)

db_request.find_service = pretend.call_recorder(
lambda iface, name=None, context=None: {
IFileStorage: {
"archive": pretend.stub(),
"cache": pretend.stub(),
},
IMetricsService: {None: metrics},
}[iface][name]
)
db_request.registry.settings["files.url"] = (
"https://files.example.com/packages/{path}"
)

assert backfillable_file.metadata_file_unbackfillable is False

metadata_backfill_individual(db_request, backfillable_file.id)

assert backfillable_file.metadata_file_unbackfillable is True
assert metrics.increment.calls == []


def test_metadata_backfill_file_delorted(db_request, monkeypatch, metrics):
stub_session = pretend.stub()
monkeypatch.setattr(warehouse.packaging.tasks, "PipSession", lambda: stub_session)
dist_from_wheel_url = pretend.call_recorder(lambda *a, **kw: None)
monkeypatch.setattr(
warehouse.packaging.tasks, "dist_from_wheel_url", dist_from_wheel_url
)
db_request.find_service = pretend.call_recorder(
lambda iface, name=None, context=None: {
IFileStorage: {
"archive": pretend.stub(),
"cache": pretend.stub(),
},
IMetricsService: {None: metrics},
}[iface][name]
)
db_request.registry.settings["files.url"] = (
"https://files.example.com/packages/{path}"
)

metadata_backfill_individual(db_request, "66642069-0000-0000-0000-000000000000")

assert dist_from_wheel_url.calls == []
assert metrics.increment.calls == []
4 changes: 0 additions & 4 deletions warehouse/packaging/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
check_file_cache_tasks_outstanding,
compute_2fa_metrics,
compute_packaging_metrics,
metadata_backfill,
update_description_html,
)
from warehouse.rate_limiting import IRateLimiter, RateLimit
Expand Down Expand Up @@ -194,6 +193,3 @@ def includeme(config):
# TODO: restore this
# if config.get_settings().get("warehouse.release_files_table"):
# config.add_periodic_task(crontab(minute=0), sync_bigquery_release_files)

# Backfill wheel metadata
config.add_periodic_task(crontab(minute="*/5"), metadata_backfill)
Loading

0 comments on commit 2e4117d

Please sign in to comment.