Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add unassign and deprecate #219

Merged
merged 36 commits into from
Aug 19, 2022
Merged
Show file tree
Hide file tree
Changes from 15 commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
c148312
WIP: unassign
aguschin Jul 25, 2022
2dac70b
first draft
aguschin Jul 18, 2022
c817ccb
WIP: introduce BaseEvent and subclasses, VStage, Version
aguschin Jul 26, 2022
f0ae6c9
fix 'gto show name'
aguschin Jul 27, 2022
aad9c42
fix 'gto history'
aguschin Jul 27, 2022
bf413f2
fix 'gto check-ref'
aguschin Jul 27, 2022
e5c30b3
adding unassign to 'history' and enabling it in 'show model'
aguschin Jul 29, 2022
512f96f
filter out lw tags later to speed up the git tags parsing
aguschin Jul 29, 2022
0fd8e97
add 'last-assignments-per-version' and 'last-versions-per-stage' flags
aguschin Jul 29, 2022
63f8310
fix 'which' and add --av and --vs
aguschin Jul 29, 2022
c5b422a
add deregister and deprecate to README
aguschin Aug 1, 2022
9b0d828
renamings: not implemented yet
aguschin Aug 1, 2022
c9fb787
more details in readme + register an artifact
aguschin Aug 1, 2022
7ee299f
add summary for GS
aguschin Aug 1, 2022
973d44c
minor readme update
aguschin Aug 2, 2022
58780f1
fix tests
aguschin Aug 4, 2022
ef2430c
add deregister and parse git tags with regexp
aguschin Aug 5, 2022
020b133
merge main into this branch
aguschin Aug 5, 2022
2cec511
fix test, update readme
aguschin Aug 5, 2022
bc40896
Update README.md
aguschin Aug 5, 2022
5fad08e
fix readme
aguschin Aug 5, 2022
1c9a604
Update README.md
aguschin Aug 8, 2022
5c1f105
update README with command
aguschin Aug 16, 2022
63d67ef
Merge branch 'feature/unassign' of https://github.com/iterative/gto i…
aguschin Aug 16, 2022
9e29dfb
fix couple of things in README
aguschin Aug 16, 2022
945d97a
WIP: add 'tag' instead of 'register' and 'promote', add 'deprecate'
aguschin Aug 16, 2022
2d094c9
add explanations to creating git tags - update README
aguschin Aug 17, 2022
df75fb2
implement --delete
aguschin Aug 17, 2022
e3f829f
add 'git push' suggestion
aguschin Aug 17, 2022
982fc60
escape '!' in git commands suggestions
aguschin Aug 17, 2022
65c3df6
delete tag2 tests
aguschin Aug 17, 2022
f070ab0
readme update: use deprecate instead of unassign and deregister
aguschin Aug 18, 2022
6098248
fix tests and add --force to register
aguschin Aug 19, 2022
ac45939
return deregister and unassign to API: it's clearer
aguschin Aug 19, 2022
76dec51
rename deregister to unregister
aguschin Aug 19, 2022
b5e7b7c
fix deprecate in CLI
aguschin Aug 19, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
231 changes: 172 additions & 59 deletions README.md

Large diffs are not rendered by default.

264 changes: 162 additions & 102 deletions gto/api.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,20 @@
import warnings
from collections import OrderedDict
from datetime import datetime
from typing import List, Optional, Union

from funcy import distinct
from git import Repo

from gto.constants import NAME, STAGE, VERSION, Event
from gto.exceptions import NoRepo, WrongArgs
from gto.constants import (
ARTIFACT,
ASSIGNMENTS_PER_VERSION,
COMMIT,
NAME,
STAGE,
VERSION,
VERSIONS_PER_STAGE,
)
from gto.exceptions import NoRepo, NotImplementedInGTO, WrongArgs
from gto.ext import EnrichmentInfo
from gto.index import (
EnrichmentManager,
Expand Down Expand Up @@ -102,6 +110,42 @@ def register(
)


def promote(
repo: Union[str, Repo],
name: str,
stage: str,
promote_version: str = None,
promote_ref: str = None,
name_version: str = None,
message: str = None,
simple: bool = False,
force: bool = False,
skip_registration: bool = False,
stdout: bool = False,
author: Optional[str] = None,
author_email: Optional[str] = None,
):
"""Assign stage to specific artifact version"""
warnings.warn(
"`gto.api.promote` is deprecated and will be removed in future releases.",
category=DeprecationWarning,
)
return GitRegistry.from_repo(repo).assign(
name,
stage,
promote_version,
promote_ref,
name_version,
message=message,
simple=simple,
force=force,
skip_registration=skip_registration,
stdout=stdout,
author=author,
author_email=author_email,
)


def assign(
repo: Union[str, Repo],
name: str,
Expand Down Expand Up @@ -134,6 +178,36 @@ def assign(
)


def unassign(
repo: Union[str, Repo],
name: str,
stage: str,
version: str = None,
ref: str = None,
message: str = None,
simple: bool = False,
force: bool = False,
delete: bool = False,
stdout: bool = False,
author: Optional[str] = None,
author_email: Optional[str] = None,
):
"""Assign stage to specific artifact version"""
return GitRegistry.from_repo(repo).unassign(
name,
stage,
version,
ref,
message=message,
simple=simple,
force=force,
delete=delete,
stdout=stdout,
author=author,
author_email=author_email,
)


def parse_tag(name: str):
return parse_tag_name(name)

Expand All @@ -152,12 +226,18 @@ def find_versions_in_stage(
repo: Union[str, Repo],
name: str,
stage: str,
all: bool = False,
assignments_per_version=ASSIGNMENTS_PER_VERSION,
versions_per_stage=VERSIONS_PER_STAGE,
registered_only: bool = False,
):
"""Return version of artifact with specific stage active"""
return GitRegistry.from_repo(repo).which(
name, stage, raise_if_not_found=False, all=all, registered_only=registered_only
name,
stage,
raise_if_not_found=False,
assignments_per_version=assignments_per_version,
versions_per_stage=versions_per_stage,
registered_only=registered_only,
)


Expand All @@ -174,7 +254,8 @@ def show(
all_commits=False,
truncate_hexsha=False,
registered_only=False,
last_stage=False,
assignments_per_version=ASSIGNMENTS_PER_VERSION,
versions_per_stage=1,
table: bool = False,
):
return (
Expand All @@ -184,7 +265,8 @@ def show(
all_branches=all_branches,
all_commits=all_commits,
registered_only=registered_only,
last_stage=last_stage,
assignments_per_version=assignments_per_version,
versions_per_stage=versions_per_stage,
table=table,
truncate_hexsha=truncate_hexsha,
)
Expand All @@ -194,7 +276,8 @@ def show(
all_branches=all_branches,
all_commits=all_commits,
registered_only=registered_only,
last_stage=last_stage,
assignments_per_version=assignments_per_version,
versions_per_stage=versions_per_stage,
table=table,
truncate_hexsha=truncate_hexsha,
)
Expand All @@ -206,7 +289,8 @@ def _show_registry(
all_branches=False,
all_commits=False,
registered_only=False,
last_stage: bool = False,
assignments_per_version: int = ASSIGNMENTS_PER_VERSION,
versions_per_stage: int = VERSIONS_PER_STAGE,
table: bool = False,
truncate_hexsha: bool = False,
):
Expand All @@ -218,19 +302,25 @@ def format_hexsha(hexsha):
reg = GitRegistry.from_repo(repo)
stages = list(reg.get_stages())
models_state = {
o.name: {
"version": format_hexsha(o.get_latest_version(registered_only=True).name)
o.artifact: {
"version": format_hexsha(o.get_latest_version(registered_only=True).version)
if o.get_latest_version(registered_only=True)
else None,
"stage": {
name: format_hexsha(
o.get_assignments(
registered_only=registered_only, last_stage=last_stage
)[name][0].version
name: ", ".join(
[
format_hexsha(s.version)
for s in o.get_vstages(
registered_only=registered_only,
assignments_per_version=assignments_per_version,
versions_per_stage=versions_per_stage,
)[name]
]
)
if name
in o.get_assignments(
registered_only=registered_only, last_stage=last_stage
in o.get_vstages(
registered_only=registered_only,
assignments_per_version=assignments_per_version,
)
else None
for name in stages
Expand Down Expand Up @@ -259,7 +349,8 @@ def _show_versions( # pylint: disable=too-many-locals
all_branches=False,
all_commits=False,
registered_only=False,
last_stage: bool = False,
assignments_per_version: int = None,
versions_per_stage: int = None,
table: bool = False,
truncate_hexsha: bool = False,
):
Expand All @@ -271,45 +362,50 @@ def format_hexsha(hexsha):
reg = GitRegistry.from_repo(repo)
if raw:
return reg.find_artifact(name).versions
versions = [
v.dict_status()
for v in reg.find_artifact(
name,
all_branches=all_branches,
all_commits=all_commits,
).get_versions(
include_non_explicit=not registered_only, include_discovered=True
)
]

artifact = reg.find_artifact(
name,
all_branches=all_branches,
all_commits=all_commits,
)
stages = artifact.get_vstages(
registered_only=registered_only,
assignments_per_version=assignments_per_version,
versions_per_stage=versions_per_stage,
)
versions = []
for v in artifact.get_versions(
include_non_explicit=not registered_only, include_discovered=True
):
v = v.dict_state()
v["stages"] = [
vstage.dict_state()
for vstages in stages.values()
for vstage in vstages
if vstage.version == v["version"]
]
versions.append(v)

if not table:
return versions

first_keys = ["artifact", "version", "stage"]
versions_ = []
for v in versions:
v["version"] = format_hexsha(v["name"])
v["version"] = format_hexsha(v["version"])
v["stage"] = ", ".join(
distinct(
s["stage"]
for s in (
v["assignments"][::-1][:1] if last_stage else v["assignments"][::-1]
)
distinct( # TODO: remove? no longer necessary
s["stage"] for s in v["stages"]
)
)
v["commit_hexsha"] = format_hexsha(v["commit_hexsha"])
v["ref"] = v["tag"] or v["commit_hexsha"]
for key in (
"enrichments",
"discovered",
"tag",
"commit_hexsha",
"name",
"message",
"author_email",
"assignments",
):
v.pop(key)
# v["enrichments"] = [e["source"] for e in v["enrichments"]]
if len(v["registrations"]) > 1:
raise NotImplementedInGTO(
"Multiple registrations are not supported currently. How you got in here?"
)
for key in list(v.keys()):
if key not in first_keys + ["created_at", "ref"]:
del v[key]
v = OrderedDict(
[(key, v[key]) for key in first_keys]
+ [(key, v[key]) for key in v if key not in first_keys]
Expand All @@ -332,7 +428,7 @@ def describe(
raise NotImplementedError


def history( # pylint: disable=too-many-locals
def history(
repo: Union[str, Repo],
artifact: str = None,
# action: str = None,
Expand All @@ -352,62 +448,27 @@ def history( # pylint: disable=too-many-locals
def format_hexsha(hexsha):
return hexsha[:7] if truncate_hexsha else hexsha

commits = [
OrderedDict(
timestamp=datetime.fromtimestamp(
reg.repo.commit(v.commit_hexsha).committed_date
),
artifact=o.name,
event=Event.COMMIT,
commit=format_hexsha(v.commit_hexsha),
author=reg.repo.commit(v.commit_hexsha).author.name,
author_email=reg.repo.commit(v.commit_hexsha).author.email,
message=reg.repo.commit(v.commit_hexsha).message,
)
for o in artifacts.values()
for v in o.get_versions(include_non_explicit=True, include_discovered=True)
]

registration = [
OrderedDict(
timestamp=v.created_at,
artifact=o.name,
event=Event.REGISTRATION,
version=format_hexsha(v.name),
commit=format_hexsha(v.commit_hexsha),
author=v.author,
author_email=v.author_email,
message=v.message,
# enrichments=[e.source for e in v.enrichments],
)
for o in artifacts.values()
for v in o.get_versions()
]

assignment = [
events = [
OrderedDict(
timestamp=p.created_at,
artifact=o.name,
event=Event.ASSIGNMENT,
version=format_hexsha(p.version),
stage=p.stage,
commit=format_hexsha(p.commit_hexsha),
author=p.author,
author_email=p.author_email,
message=p.message,
timestamp=e.created_at,
artifact=e.artifact,
event=type(e).__name__.lower(),
priority=e.priority,
version=format_hexsha(e.version),
stage=getattr(e, "stage", None),
commit=format_hexsha(e.commit_hexsha),
author=e.author,
author_email=e.author_email,
message=e.message,
ref=format_hexsha(e.ref) if e.ref == e.commit_hexsha else e.ref,
)
for o in artifacts.values()
for p in o.stages
for e in o.get_events()
]

events_order = {
Event.COMMIT: 1,
Event.REGISTRATION: 2,
Event.ASSIGNMENT: 3,
}
events = sorted(
commits + registration + assignment,
key=lambda x: (x["timestamp"], events_order[x["event"]]),
events,
key=lambda x: (x["timestamp"], x["priority"]),
)
if not ascending:
events.reverse()
Expand All @@ -417,13 +478,12 @@ def format_hexsha(hexsha):
return events
keys_order = [
"timestamp",
"artifact",
ARTIFACT,
"event",
VERSION,
STAGE,
# "enrichments",
"commit",
"author",
COMMIT,
"ref",
]
keys_order = [c for c in keys_order if any(c in event for event in events)]
events = [
Expand Down
Loading