diff --git a/docs/api/S3.ipynb b/docs/api/S3.ipynb index 1580ddb6..22c73cad 100644 --- a/docs/api/S3.ipynb +++ b/docs/api/S3.ipynb @@ -24,10 +24,10 @@ "id": "267ce6f4", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:55.364067Z", - "iopub.status.busy": "2024-07-25T06:16:55.363954Z", - "iopub.status.idle": "2024-07-25T06:16:55.632670Z", - "shell.execute_reply": "2024-07-25T06:16:55.632359Z" + "iopub.execute_input": "2024-11-05T04:49:38.019890Z", + "iopub.status.busy": "2024-11-05T04:49:38.019300Z", + "iopub.status.idle": "2024-11-05T04:49:38.334199Z", + "shell.execute_reply": "2024-11-05T04:49:38.333828Z" } }, "outputs": [], @@ -55,10 +55,10 @@ "id": "be77093e", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:55.635239Z", - "iopub.status.busy": "2024-07-25T06:16:55.635039Z", - "iopub.status.idle": "2024-07-25T06:16:55.655916Z", - "shell.execute_reply": "2024-07-25T06:16:55.655612Z" + "iopub.execute_input": "2024-11-05T04:49:38.336976Z", + "iopub.status.busy": "2024-11-05T04:49:38.336811Z", + "iopub.status.idle": "2024-11-05T04:49:38.358821Z", + "shell.execute_reply": "2024-11-05T04:49:38.358433Z" } }, "outputs": [ @@ -83,7 +83,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -101,10 +101,10 @@ "id": "a0e04313", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:55.658196Z", - "iopub.status.busy": "2024-07-25T06:16:55.658076Z", - "iopub.status.idle": "2024-07-25T06:16:55.660743Z", - "shell.execute_reply": "2024-07-25T06:16:55.660439Z" + "iopub.execute_input": "2024-11-05T04:49:38.361347Z", + "iopub.status.busy": "2024-11-05T04:49:38.361197Z", + "iopub.status.idle": "2024-11-05T04:49:38.364105Z", + "shell.execute_reply": "2024-11-05T04:49:38.363777Z" } }, "outputs": [ @@ -112,9 +112,9 @@ "data": { "text/html": [ "\n", - "

method S3.close (self)[source]

Delete all temporary files downloaded in this context.

\n", + "

method S3.close (self)[source]

Delete all temporary files downloaded in this context.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -123,7 +123,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 3, @@ -149,10 +149,10 @@ "id": "30c3afdf", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:55.663324Z", - "iopub.status.busy": "2024-07-25T06:16:55.663117Z", - "iopub.status.idle": "2024-07-25T06:16:55.666684Z", - "shell.execute_reply": "2024-07-25T06:16:55.666412Z" + "iopub.execute_input": "2024-11-05T04:49:38.366890Z", + "iopub.status.busy": "2024-11-05T04:49:38.366757Z", + "iopub.status.idle": "2024-11-05T04:49:38.370665Z", + "shell.execute_reply": "2024-11-05T04:49:38.370315Z" } }, "outputs": [ @@ -160,9 +160,9 @@ "data": { "text/html": [ "\n", - "

method S3.get (self, key: Union[str, metaflow.plugins.datatools.s3.s3.S3GetObject, NoneType] = None, return_missing: bool = False, return_info: bool = True) -> metaflow.plugins.datatools.s3.s3.S3Object[source]

Get a single object from S3.

Parameters
----------
key : Union[str, S3GetObject], optional, default None
    Object to download. It can be an S3 url, a path suffix, or
    an S3GetObject that defines a range of data to download. If None, or
    not provided, gets the S3 root.
return_missing : bool, default False
    If set to True, do not raise an exception for a missing key but
    return it as an `S3Object` with `.exists == False`.
return_info : bool, default True
    If set to True, fetch the content-type and user metadata associated
    with the object at no extra cost, included for symmetry with `get_many`

Returns
-------
S3Object
    An S3Object corresponding to the object requested.

\n", + "

method S3.get (self, key: Union[str, metaflow.plugins.datatools.s3.s3.S3GetObject, NoneType] = None, return_missing: bool = False, return_info: bool = True) -> metaflow.plugins.datatools.s3.s3.S3Object[source]

Get a single object from S3.

Parameters
----------
key : Union[str, S3GetObject], optional, default None
    Object to download. It can be an S3 url, a path suffix, or
    an S3GetObject that defines a range of data to download. If None, or
    not provided, gets the S3 root.
return_missing : bool, default False
    If set to True, do not raise an exception for a missing key but
    return it as an `S3Object` with `.exists == False`.
return_info : bool, default True
    If set to True, fetch the content-type and user metadata associated
    with the object at no extra cost, included for symmetry with `get_many`

Returns
-------
S3Object
    An S3Object corresponding to the object requested.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -178,7 +178,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 4, @@ -196,10 +196,10 @@ "id": "2bbb1e36", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:55.668637Z", - "iopub.status.busy": "2024-07-25T06:16:55.668562Z", - "iopub.status.idle": "2024-07-25T06:16:55.671501Z", - "shell.execute_reply": "2024-07-25T06:16:55.671249Z" + "iopub.execute_input": "2024-11-05T04:49:38.373996Z", + "iopub.status.busy": "2024-11-05T04:49:38.373781Z", + "iopub.status.idle": "2024-11-05T04:49:38.377091Z", + "shell.execute_reply": "2024-11-05T04:49:38.376813Z" } }, "outputs": [ @@ -207,9 +207,9 @@ "data": { "text/html": [ "\n", - "

method S3.get_many (self, keys: Iterable[Union[str, metaflow.plugins.datatools.s3.s3.S3GetObject]], return_missing: bool = False, return_info: bool = True) -> List[metaflow.plugins.datatools.s3.s3.S3Object][source]

Get many objects from S3 in parallel.

Parameters
----------
keys : Iterable[Union[str, S3GetObject]]
    Objects to download. Each object can be an S3 url, a path suffix, or
    an S3GetObject that defines a range of data to download.
return_missing : bool, default False
    If set to True, do not raise an exception for a missing key but
    return it as an `S3Object` with `.exists == False`.
return_info : bool, default True
    If set to True, fetch the content-type and user metadata associated
    with the object at no extra cost, included for symmetry with `get_many`.

Returns
-------
List[S3Object]
    S3Objects corresponding to the objects requested.

\n", + "

method S3.get_many (self, keys: Iterable[Union[str, metaflow.plugins.datatools.s3.s3.S3GetObject]], return_missing: bool = False, return_info: bool = True) -> List[metaflow.plugins.datatools.s3.s3.S3Object][source]

Get many objects from S3 in parallel.

Parameters
----------
keys : Iterable[Union[str, S3GetObject]]
    Objects to download. Each object can be an S3 url, a path suffix, or
    an S3GetObject that defines a range of data to download.
return_missing : bool, default False
    If set to True, do not raise an exception for a missing key but
    return it as an `S3Object` with `.exists == False`.
return_info : bool, default True
    If set to True, fetch the content-type and user metadata associated
    with the object at no extra cost, included for symmetry with `get_many`.

Returns
-------
List[S3Object]
    S3Objects corresponding to the objects requested.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -225,7 +225,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 5, @@ -243,10 +243,10 @@ "id": "64a74dcf", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:55.673609Z", - "iopub.status.busy": "2024-07-25T06:16:55.673501Z", - "iopub.status.idle": "2024-07-25T06:16:55.676324Z", - "shell.execute_reply": "2024-07-25T06:16:55.676093Z" + "iopub.execute_input": "2024-11-05T04:49:38.380210Z", + "iopub.status.busy": "2024-11-05T04:49:38.380100Z", + "iopub.status.idle": "2024-11-05T04:49:38.383182Z", + "shell.execute_reply": "2024-11-05T04:49:38.382901Z" } }, "outputs": [ @@ -254,9 +254,9 @@ "data": { "text/html": [ "\n", - "

method S3.get_recursive (self, keys: Iterable[str], return_info: bool = False) -> List[metaflow.plugins.datatools.s3.s3.S3Object][source]

Get many objects from S3 recursively in parallel.

Parameters
----------
keys : Iterable[str]
    Prefixes to download recursively. Each prefix can be an S3 url or a path suffix
    which define the root prefix under which all objects are downloaded.
return_info : bool, default False
    If set to True, fetch the content-type and user metadata associated
    with the object.

Returns
-------
List[S3Object]
    S3Objects stored under the given prefixes.

\n", + "

method S3.get_recursive (self, keys: Iterable[str], return_info: bool = False) -> List[metaflow.plugins.datatools.s3.s3.S3Object][source]

Get many objects from S3 recursively in parallel.

Parameters
----------
keys : Iterable[str]
    Prefixes to download recursively. Each prefix can be an S3 url or a path suffix
    which define the root prefix under which all objects are downloaded.
return_info : bool, default False
    If set to True, fetch the content-type and user metadata associated
    with the object.

Returns
-------
List[S3Object]
    S3Objects stored under the given prefixes.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -271,7 +271,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 6, @@ -289,10 +289,10 @@ "id": "62b5aa5c", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:55.678408Z", - "iopub.status.busy": "2024-07-25T06:16:55.678312Z", - "iopub.status.idle": "2024-07-25T06:16:55.680730Z", - "shell.execute_reply": "2024-07-25T06:16:55.680538Z" + "iopub.execute_input": "2024-11-05T04:49:38.386988Z", + "iopub.status.busy": "2024-11-05T04:49:38.386806Z", + "iopub.status.idle": "2024-11-05T04:49:38.389836Z", + "shell.execute_reply": "2024-11-05T04:49:38.389558Z" } }, "outputs": [ @@ -300,9 +300,9 @@ "data": { "text/html": [ "\n", - "

method S3.get_all (self, return_info: bool = False) -> List[metaflow.plugins.datatools.s3.s3.S3Object][source]

Get all objects under the prefix set in the `S3` constructor.

This method requires that the `S3` object is initialized either with `run` or
`s3root`.

Parameters
----------
return_info : bool, default False
    If set to True, fetch the content-type and user metadata associated
    with the object.

Returns
-------
Iterable[S3Object]
    S3Objects stored under the main prefix.

\n", + "

method S3.get_all (self, return_info: bool = False) -> List[metaflow.plugins.datatools.s3.s3.S3Object][source]

Get all objects under the prefix set in the `S3` constructor.

This method requires that the `S3` object is initialized either with `run` or
`s3root`.

Parameters
----------
return_info : bool, default False
    If set to True, fetch the content-type and user metadata associated
    with the object.

Returns
-------
Iterable[S3Object]
    S3Objects stored under the main prefix.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -316,7 +316,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 7, @@ -342,10 +342,10 @@ "id": "b00c7e64", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:55.682599Z", - "iopub.status.busy": "2024-07-25T06:16:55.682502Z", - "iopub.status.idle": "2024-07-25T06:16:55.685091Z", - "shell.execute_reply": "2024-07-25T06:16:55.684875Z" + "iopub.execute_input": "2024-11-05T04:49:38.392638Z", + "iopub.status.busy": "2024-11-05T04:49:38.392514Z", + "iopub.status.idle": "2024-11-05T04:49:38.395774Z", + "shell.execute_reply": "2024-11-05T04:49:38.395465Z" }, "scrolled": true }, @@ -354,9 +354,9 @@ "data": { "text/html": [ "\n", - "

method S3.list_paths (self, keys: Optional[Iterable[str]] = None) -> List[metaflow.plugins.datatools.s3.s3.S3Object][source]

List the next level of paths in S3.

If multiple keys are specified, listings are done in parallel. The returned
S3Objects have `.exists == False` if the path refers to a prefix, not an
existing S3 object.

For instance, if the directory hierarchy is
```
a/0.txt
a/b/1.txt
a/c/2.txt
a/d/e/3.txt
f/4.txt
```
The `list_paths(['a', 'f'])` call returns
```
a/0.txt (exists == True)
a/b/ (exists == False)
a/c/ (exists == False)
a/d/ (exists == False)
f/4.txt (exists == True)
```

Parameters
----------
keys : Iterable[str], optional, default None
    List of paths.

Returns
-------
List[S3Object]
    S3Objects under the given paths, including prefixes (directories) that
    do not correspond to leaf objects.

\n", + "

method S3.list_paths (self, keys: Optional[Iterable[str]] = None) -> List[metaflow.plugins.datatools.s3.s3.S3Object][source]

List the next level of paths in S3.

If multiple keys are specified, listings are done in parallel. The returned
S3Objects have `.exists == False` if the path refers to a prefix, not an
existing S3 object.

For instance, if the directory hierarchy is
```
a/0.txt
a/b/1.txt
a/c/2.txt
a/d/e/3.txt
f/4.txt
```
The `list_paths(['a', 'f'])` call returns
```
a/0.txt (exists == True)
a/b/ (exists == False)
a/c/ (exists == False)
a/d/ (exists == False)
f/4.txt (exists == True)
```

Parameters
----------
keys : Iterable[str], optional, default None
    List of paths.

Returns
-------
List[S3Object]
    S3Objects under the given paths, including prefixes (directories) that
    do not correspond to leaf objects.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -370,7 +370,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 8, @@ -388,10 +388,10 @@ "id": "9b39aa1f", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:55.687389Z", - "iopub.status.busy": "2024-07-25T06:16:55.687310Z", - "iopub.status.idle": "2024-07-25T06:16:55.689833Z", - "shell.execute_reply": "2024-07-25T06:16:55.689561Z" + "iopub.execute_input": "2024-11-05T04:49:38.398766Z", + "iopub.status.busy": "2024-11-05T04:49:38.398641Z", + "iopub.status.idle": "2024-11-05T04:49:38.401505Z", + "shell.execute_reply": "2024-11-05T04:49:38.401283Z" }, "scrolled": true }, @@ -400,9 +400,9 @@ "data": { "text/html": [ "\n", - "

method S3.list_recursive (self, keys: Optional[Iterable[str]] = None) -> List[metaflow.plugins.datatools.s3.s3.S3Object][source]

List all objects recursively under the given prefixes.

If multiple keys are specified, listings are done in parallel. All objects
returned have `.exists == True` as this call always returns leaf objects.

For instance, if the directory hierarchy is
```
a/0.txt
a/b/1.txt
a/c/2.txt
a/d/e/3.txt
f/4.txt
```
The `list_paths(['a', 'f'])` call returns
```
a/0.txt (exists == True)
a/b/1.txt (exists == True)
a/c/2.txt (exists == True)
a/d/e/3.txt (exists == True)
f/4.txt (exists == True)
```

Parameters
----------
keys : Iterable[str], optional, default None
    List of paths.

Returns
-------
List[S3Object]
    S3Objects under the given paths.

\n", + "

method S3.list_recursive (self, keys: Optional[Iterable[str]] = None) -> List[metaflow.plugins.datatools.s3.s3.S3Object][source]

List all objects recursively under the given prefixes.

If multiple keys are specified, listings are done in parallel. All objects
returned have `.exists == True` as this call always returns leaf objects.

For instance, if the directory hierarchy is
```
a/0.txt
a/b/1.txt
a/c/2.txt
a/d/e/3.txt
f/4.txt
```
The `list_paths(['a', 'f'])` call returns
```
a/0.txt (exists == True)
a/b/1.txt (exists == True)
a/c/2.txt (exists == True)
a/d/e/3.txt (exists == True)
f/4.txt (exists == True)
```

Parameters
----------
keys : Iterable[str], optional, default None
    List of paths.

Returns
-------
List[S3Object]
    S3Objects under the given paths.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -416,7 +416,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 9, @@ -442,10 +442,10 @@ "id": "8e76d108", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:55.692135Z", - "iopub.status.busy": "2024-07-25T06:16:55.692039Z", - "iopub.status.idle": "2024-07-25T06:16:55.695207Z", - "shell.execute_reply": "2024-07-25T06:16:55.694894Z" + "iopub.execute_input": "2024-11-05T04:49:38.404965Z", + "iopub.status.busy": "2024-11-05T04:49:38.404849Z", + "iopub.status.idle": "2024-11-05T04:49:38.408438Z", + "shell.execute_reply": "2024-11-05T04:49:38.408076Z" }, "scrolled": true }, @@ -454,9 +454,9 @@ "data": { "text/html": [ "\n", - "

method S3.put (self, key: Union[str, metaflow.plugins.datatools.s3.s3.S3PutObject], obj: Union[io.RawIOBase, io.BufferedIOBase, str, bytes], overwrite: bool = True, content_type: Optional[str] = None, metadata: Optional[Dict[str, str]] = None) -> str[source]

Upload a single object to S3.

Parameters
----------
key : Union[str, S3PutObject]
    Object path. It can be an S3 url or a path suffix.
obj : PutValue
    An object to store in S3. Strings are converted to UTF-8 encoding.
overwrite : bool, default True
    Overwrite the object if it exists. If set to False, the operation
    succeeds without uploading anything if the key already exists.
content_type : str, optional, default None
    Optional MIME type for the object.
metadata : Dict[str, str], optional, default None
    A JSON-encodable dictionary of additional headers to be stored
    as metadata with the object.

Returns
-------
str
    URL of the object stored.

\n", + "

method S3.put (self, key: Union[str, metaflow.plugins.datatools.s3.s3.S3PutObject], obj: Union[io.RawIOBase, io.BufferedIOBase, str, bytes], overwrite: bool = True, content_type: Optional[str] = None, metadata: Optional[Dict[str, str]] = None) -> str[source]

Upload a single object to S3.

Parameters
----------
key : Union[str, S3PutObject]
    Object path. It can be an S3 url or a path suffix.
obj : PutValue
    An object to store in S3. Strings are converted to UTF-8 encoding.
overwrite : bool, default True
    Overwrite the object if it exists. If set to False, the operation
    succeeds without uploading anything if the key already exists.
content_type : str, optional, default None
    Optional MIME type for the object.
metadata : Dict[str, str], optional, default None
    A JSON-encodable dictionary of additional headers to be stored
    as metadata with the object.

Returns
-------
str
    URL of the object stored.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -474,7 +474,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 10, @@ -492,10 +492,10 @@ "id": "ae5c6247", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:55.697366Z", - "iopub.status.busy": "2024-07-25T06:16:55.697267Z", - "iopub.status.idle": "2024-07-25T06:16:55.700234Z", - "shell.execute_reply": "2024-07-25T06:16:55.699996Z" + "iopub.execute_input": "2024-11-05T04:49:38.410727Z", + "iopub.status.busy": "2024-11-05T04:49:38.410607Z", + "iopub.status.idle": "2024-11-05T04:49:38.413901Z", + "shell.execute_reply": "2024-11-05T04:49:38.413609Z" } }, "outputs": [ @@ -503,9 +503,9 @@ "data": { "text/html": [ "\n", - "

method S3.put_many (self, key_objs: List[Union[Tuple[str, Union[io.RawIOBase, io.BufferedIOBase, str, bytes]], metaflow.plugins.datatools.s3.s3.S3PutObject]], overwrite: bool = True) -> List[Tuple[str, str]][source]

Upload many objects to S3.

Each object to be uploaded can be specified in two ways:

1. As a `(key, obj)` tuple where `key` is a string specifying
   the path and `obj` is a string or a bytes object.

2. As a `S3PutObject` which contains additional metadata to be
   stored with the object.

Parameters
----------
key_objs : List[Union[Tuple[str, PutValue], S3PutObject]]
    List of key-object pairs to upload.
overwrite : bool, default True
    Overwrite the object if it exists. If set to False, the operation
    succeeds without uploading anything if the key already exists.

Returns
-------
List[Tuple[str, str]]
    List of `(key, url)` pairs corresponding to the objects uploaded.

\n", + "

method S3.put_many (self, key_objs: List[Union[Tuple[str, Union[io.RawIOBase, io.BufferedIOBase, str, bytes]], metaflow.plugins.datatools.s3.s3.S3PutObject]], overwrite: bool = True) -> List[Tuple[str, str]][source]

Upload many objects to S3.

Each object to be uploaded can be specified in two ways:

1. As a `(key, obj)` tuple where `key` is a string specifying
   the path and `obj` is a string or a bytes object.

2. As a `S3PutObject` which contains additional metadata to be
   stored with the object.

Parameters
----------
key_objs : List[Union[Tuple[str, PutValue], S3PutObject]]
    List of key-object pairs to upload.
overwrite : bool, default True
    Overwrite the object if it exists. If set to False, the operation
    succeeds without uploading anything if the key already exists.

Returns
-------
List[Tuple[str, str]]
    List of `(key, url)` pairs corresponding to the objects uploaded.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -520,7 +520,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 11, @@ -538,10 +538,10 @@ "id": "d9c3ea39", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:55.702286Z", - "iopub.status.busy": "2024-07-25T06:16:55.702189Z", - "iopub.status.idle": "2024-07-25T06:16:55.704955Z", - "shell.execute_reply": "2024-07-25T06:16:55.704709Z" + "iopub.execute_input": "2024-11-05T04:49:38.416467Z", + "iopub.status.busy": "2024-11-05T04:49:38.416346Z", + "iopub.status.idle": "2024-11-05T04:49:38.419605Z", + "shell.execute_reply": "2024-11-05T04:49:38.419279Z" } }, "outputs": [ @@ -549,9 +549,9 @@ "data": { "text/html": [ "\n", - "

method S3.put_files (self, key_paths: List[Union[Tuple[str, Union[io.RawIOBase, io.BufferedIOBase, str, bytes]], metaflow.plugins.datatools.s3.s3.S3PutObject]], overwrite: bool = True) -> List[Tuple[str, str]][source]

Upload many local files to S3.

Each file to be uploaded can be specified in two ways:

1. As a `(key, path)` tuple where `key` is a string specifying
   the S3 path and `path` is the path to a local file.

2. As a `S3PutObject` which contains additional metadata to be
   stored with the file.

Parameters
----------
key_paths :  List[Union[Tuple[str, PutValue], S3PutObject]]
    List of files to upload.
overwrite : bool, default True
    Overwrite the object if it exists. If set to False, the operation
    succeeds without uploading anything if the key already exists.

Returns
-------
List[Tuple[str, str]]
    List of `(key, url)` pairs corresponding to the files uploaded.

\n", + "

method S3.put_files (self, key_paths: List[Union[Tuple[str, Union[io.RawIOBase, io.BufferedIOBase, str, bytes]], metaflow.plugins.datatools.s3.s3.S3PutObject]], overwrite: bool = True) -> List[Tuple[str, str]][source]

Upload many local files to S3.

Each file to be uploaded can be specified in two ways:

1. As a `(key, path)` tuple where `key` is a string specifying
   the S3 path and `path` is the path to a local file.

2. As a `S3PutObject` which contains additional metadata to be
   stored with the file.

Parameters
----------
key_paths :  List[Union[Tuple[str, PutValue], S3PutObject]]
    List of files to upload.
overwrite : bool, default True
    Overwrite the object if it exists. If set to False, the operation
    succeeds without uploading anything if the key already exists.

Returns
-------
List[Tuple[str, str]]
    List of `(key, url)` pairs corresponding to the files uploaded.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -566,7 +566,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 12, @@ -592,10 +592,10 @@ "id": "c7055621", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:55.706871Z", - "iopub.status.busy": "2024-07-25T06:16:55.706784Z", - "iopub.status.idle": "2024-07-25T06:16:55.709603Z", - "shell.execute_reply": "2024-07-25T06:16:55.709377Z" + "iopub.execute_input": "2024-11-05T04:49:38.422113Z", + "iopub.status.busy": "2024-11-05T04:49:38.422021Z", + "iopub.status.idle": "2024-11-05T04:49:38.425162Z", + "shell.execute_reply": "2024-11-05T04:49:38.424790Z" } }, "outputs": [ @@ -603,9 +603,9 @@ "data": { "text/html": [ "\n", - "

method S3.info (self, key: Optional[str] = None, return_missing: bool = False) -> metaflow.plugins.datatools.s3.s3.S3Object[source]

Get metadata about a single object in S3.

This call makes a single `HEAD` request to S3 which can be
much faster than downloading all data with `get`.

Parameters
----------
key : str, optional, default None
    Object to query. It can be an S3 url or a path suffix.
return_missing : bool, default False
    If set to True, do not raise an exception for a missing key but
    return it as an `S3Object` with `.exists == False`.

Returns
-------
S3Object
    An S3Object corresponding to the object requested. The object
    will have `.downloaded == False`.

\n", + "

method S3.info (self, key: Optional[str] = None, return_missing: bool = False) -> metaflow.plugins.datatools.s3.s3.S3Object[source]

Get metadata about a single object in S3.

This call makes a single `HEAD` request to S3 which can be
much faster than downloading all data with `get`.

Parameters
----------
key : str, optional, default None
    Object to query. It can be an S3 url or a path suffix.
return_missing : bool, default False
    If set to True, do not raise an exception for a missing key but
    return it as an `S3Object` with `.exists == False`.

Returns
-------
S3Object
    An S3Object corresponding to the object requested. The object
    will have `.downloaded == False`.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -620,7 +620,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 13, @@ -638,10 +638,10 @@ "id": "ba5e0f9b", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:55.711568Z", - "iopub.status.busy": "2024-07-25T06:16:55.711473Z", - "iopub.status.idle": "2024-07-25T06:16:55.714093Z", - "shell.execute_reply": "2024-07-25T06:16:55.713851Z" + "iopub.execute_input": "2024-11-05T04:49:38.428210Z", + "iopub.status.busy": "2024-11-05T04:49:38.427994Z", + "iopub.status.idle": "2024-11-05T04:49:38.431299Z", + "shell.execute_reply": "2024-11-05T04:49:38.430995Z" } }, "outputs": [ @@ -649,9 +649,9 @@ "data": { "text/html": [ "\n", - "

method S3.info_many (self, keys: Iterable[str], return_missing: bool = False) -> List[metaflow.plugins.datatools.s3.s3.S3Object][source]

Get metadata about many objects in S3 in parallel.

This call makes a single `HEAD` request to S3 which can be
much faster than downloading all data with `get`.

Parameters
----------
keys : Iterable[str]
    Objects to query. Each key can be an S3 url or a path suffix.
return_missing : bool, default False
    If set to True, do not raise an exception for a missing key but
    return it as an `S3Object` with `.exists == False`.

Returns
-------
List[S3Object]
    A list of S3Objects corresponding to the paths requested. The
    objects will have `.downloaded == False`.

\n", + "

method S3.info_many (self, keys: Iterable[str], return_missing: bool = False) -> List[metaflow.plugins.datatools.s3.s3.S3Object][source]

Get metadata about many objects in S3 in parallel.

This call makes a single `HEAD` request to S3 which can be
much faster than downloading all data with `get`.

Parameters
----------
keys : Iterable[str]
    Objects to query. Each key can be an S3 url or a path suffix.
return_missing : bool, default False
    If set to True, do not raise an exception for a missing key but
    return it as an `S3Object` with `.exists == False`.

Returns
-------
List[S3Object]
    A list of S3Objects corresponding to the paths requested. The
    objects will have `.downloaded == False`.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -666,7 +666,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 14, @@ -696,10 +696,10 @@ "id": "788746a9", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:55.716393Z", - "iopub.status.busy": "2024-07-25T06:16:55.716303Z", - "iopub.status.idle": "2024-07-25T06:16:55.724697Z", - "shell.execute_reply": "2024-07-25T06:16:55.724457Z" + "iopub.execute_input": "2024-11-05T04:49:38.433624Z", + "iopub.status.busy": "2024-11-05T04:49:38.433500Z", + "iopub.status.idle": "2024-11-05T04:49:38.441946Z", + "shell.execute_reply": "2024-11-05T04:49:38.441678Z" }, "scrolled": true }, @@ -719,7 +719,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 15, @@ -737,10 +737,10 @@ "id": "d9e06053", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:55.726726Z", - "iopub.status.busy": "2024-07-25T06:16:55.726637Z", - "iopub.status.idle": "2024-07-25T06:16:55.728693Z", - "shell.execute_reply": "2024-07-25T06:16:55.728433Z" + "iopub.execute_input": "2024-11-05T04:49:38.444510Z", + "iopub.status.busy": "2024-11-05T04:49:38.444389Z", + "iopub.status.idle": "2024-11-05T04:49:38.446647Z", + "shell.execute_reply": "2024-11-05T04:49:38.446399Z" }, "scrolled": false }, @@ -760,7 +760,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 16, @@ -778,10 +778,10 @@ "id": "a6b91a6f", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:55.731022Z", - "iopub.status.busy": "2024-07-25T06:16:55.730936Z", - "iopub.status.idle": "2024-07-25T06:16:55.732901Z", - "shell.execute_reply": "2024-07-25T06:16:55.732647Z" + "iopub.execute_input": "2024-11-05T04:49:38.449859Z", + "iopub.status.busy": "2024-11-05T04:49:38.449748Z", + "iopub.status.idle": "2024-11-05T04:49:38.452006Z", + "shell.execute_reply": "2024-11-05T04:49:38.451700Z" }, "scrolled": false }, @@ -801,7 +801,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 17, @@ -819,10 +819,10 @@ "id": "7c4203ec", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:55.735121Z", - "iopub.status.busy": "2024-07-25T06:16:55.735033Z", - "iopub.status.idle": "2024-07-25T06:16:55.737100Z", - "shell.execute_reply": "2024-07-25T06:16:55.736876Z" + "iopub.execute_input": "2024-11-05T04:49:38.455302Z", + "iopub.status.busy": "2024-11-05T04:49:38.455081Z", + "iopub.status.idle": "2024-11-05T04:49:38.457335Z", + "shell.execute_reply": "2024-11-05T04:49:38.457040Z" }, "scrolled": false }, @@ -842,7 +842,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 18, @@ -860,10 +860,10 @@ "id": "e84a384d", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:55.739260Z", - "iopub.status.busy": "2024-07-25T06:16:55.739170Z", - "iopub.status.idle": "2024-07-25T06:16:55.741240Z", - "shell.execute_reply": "2024-07-25T06:16:55.740994Z" + "iopub.execute_input": "2024-11-05T04:49:38.460294Z", + "iopub.status.busy": "2024-11-05T04:49:38.460183Z", + "iopub.status.idle": "2024-11-05T04:49:38.462202Z", + "shell.execute_reply": "2024-11-05T04:49:38.461984Z" }, "scrolled": false }, @@ -883,7 +883,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 19, @@ -901,10 +901,10 @@ "id": "eeb28c6b", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:55.743302Z", - "iopub.status.busy": "2024-07-25T06:16:55.743199Z", - "iopub.status.idle": "2024-07-25T06:16:55.745316Z", - "shell.execute_reply": "2024-07-25T06:16:55.745046Z" + "iopub.execute_input": "2024-11-05T04:49:38.465462Z", + "iopub.status.busy": "2024-11-05T04:49:38.465343Z", + "iopub.status.idle": "2024-11-05T04:49:38.467662Z", + "shell.execute_reply": "2024-11-05T04:49:38.467381Z" }, "scrolled": false }, @@ -924,7 +924,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 20, @@ -942,10 +942,10 @@ "id": "71ae77e4", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:55.747389Z", - "iopub.status.busy": "2024-07-25T06:16:55.747309Z", - "iopub.status.idle": "2024-07-25T06:16:55.749317Z", - "shell.execute_reply": "2024-07-25T06:16:55.749047Z" + "iopub.execute_input": "2024-11-05T04:49:38.470791Z", + "iopub.status.busy": "2024-11-05T04:49:38.470669Z", + "iopub.status.idle": "2024-11-05T04:49:38.473078Z", + "shell.execute_reply": "2024-11-05T04:49:38.472844Z" }, "scrolled": true }, @@ -965,7 +965,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 21, @@ -983,10 +983,10 @@ "id": "2be25e64", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:55.751525Z", - "iopub.status.busy": "2024-07-25T06:16:55.751441Z", - "iopub.status.idle": "2024-07-25T06:16:55.753445Z", - "shell.execute_reply": "2024-07-25T06:16:55.753202Z" + "iopub.execute_input": "2024-11-05T04:49:38.476108Z", + "iopub.status.busy": "2024-11-05T04:49:38.476002Z", + "iopub.status.idle": "2024-11-05T04:49:38.478366Z", + "shell.execute_reply": "2024-11-05T04:49:38.478003Z" }, "scrolled": true }, @@ -1006,7 +1006,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 22, @@ -1024,10 +1024,10 @@ "id": "4f20cd6a", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:55.755533Z", - "iopub.status.busy": "2024-07-25T06:16:55.755449Z", - "iopub.status.idle": "2024-07-25T06:16:55.757428Z", - "shell.execute_reply": "2024-07-25T06:16:55.757176Z" + "iopub.execute_input": "2024-11-05T04:49:38.481425Z", + "iopub.status.busy": "2024-11-05T04:49:38.481297Z", + "iopub.status.idle": "2024-11-05T04:49:38.483512Z", + "shell.execute_reply": "2024-11-05T04:49:38.483232Z" }, "scrolled": false }, @@ -1047,7 +1047,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 23, @@ -1065,10 +1065,10 @@ "id": "86be736c", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:55.759514Z", - "iopub.status.busy": "2024-07-25T06:16:55.759426Z", - "iopub.status.idle": "2024-07-25T06:16:55.761395Z", - "shell.execute_reply": "2024-07-25T06:16:55.761162Z" + "iopub.execute_input": "2024-11-05T04:49:38.486507Z", + "iopub.status.busy": "2024-11-05T04:49:38.486394Z", + "iopub.status.idle": "2024-11-05T04:49:38.488532Z", + "shell.execute_reply": "2024-11-05T04:49:38.488235Z" }, "scrolled": true }, @@ -1088,7 +1088,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 24, @@ -1106,10 +1106,10 @@ "id": "ac57517e", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:55.763551Z", - "iopub.status.busy": "2024-07-25T06:16:55.763464Z", - "iopub.status.idle": "2024-07-25T06:16:55.765486Z", - "shell.execute_reply": "2024-07-25T06:16:55.765246Z" + "iopub.execute_input": "2024-11-05T04:49:38.491976Z", + "iopub.status.busy": "2024-11-05T04:49:38.491865Z", + "iopub.status.idle": "2024-11-05T04:49:38.494001Z", + "shell.execute_reply": "2024-11-05T04:49:38.493790Z" }, "scrolled": false }, @@ -1129,7 +1129,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 25, @@ -1147,10 +1147,10 @@ "id": "c73642bb", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:55.767601Z", - "iopub.status.busy": "2024-07-25T06:16:55.767518Z", - "iopub.status.idle": "2024-07-25T06:16:55.769503Z", - "shell.execute_reply": "2024-07-25T06:16:55.769254Z" + "iopub.execute_input": "2024-11-05T04:49:38.496928Z", + "iopub.status.busy": "2024-11-05T04:49:38.496818Z", + "iopub.status.idle": "2024-11-05T04:49:38.498935Z", + "shell.execute_reply": "2024-11-05T04:49:38.498663Z" }, "scrolled": true }, @@ -1170,7 +1170,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 26, @@ -1188,10 +1188,10 @@ "id": "82f36246", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:55.771721Z", - "iopub.status.busy": "2024-07-25T06:16:55.771626Z", - "iopub.status.idle": "2024-07-25T06:16:55.773673Z", - "shell.execute_reply": "2024-07-25T06:16:55.773403Z" + "iopub.execute_input": "2024-11-05T04:49:38.502090Z", + "iopub.status.busy": "2024-11-05T04:49:38.501948Z", + "iopub.status.idle": "2024-11-05T04:49:38.504259Z", + "shell.execute_reply": "2024-11-05T04:49:38.503978Z" }, "scrolled": true }, @@ -1211,7 +1211,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 27, @@ -1229,10 +1229,10 @@ "id": "1c06cf68", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:55.775747Z", - "iopub.status.busy": "2024-07-25T06:16:55.775653Z", - "iopub.status.idle": "2024-07-25T06:16:55.777621Z", - "shell.execute_reply": "2024-07-25T06:16:55.777370Z" + "iopub.execute_input": "2024-11-05T04:49:38.507548Z", + "iopub.status.busy": "2024-11-05T04:49:38.507426Z", + "iopub.status.idle": "2024-11-05T04:49:38.509705Z", + "shell.execute_reply": "2024-11-05T04:49:38.509448Z" }, "scrolled": false }, @@ -1249,7 +1249,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 28, @@ -1267,10 +1267,10 @@ "id": "22068e78", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:55.779811Z", - "iopub.status.busy": "2024-07-25T06:16:55.779711Z", - "iopub.status.idle": "2024-07-25T06:16:55.781811Z", - "shell.execute_reply": "2024-07-25T06:16:55.781562Z" + "iopub.execute_input": "2024-11-05T04:49:38.513155Z", + "iopub.status.busy": "2024-11-05T04:49:38.513045Z", + "iopub.status.idle": "2024-11-05T04:49:38.515174Z", + "shell.execute_reply": "2024-11-05T04:49:38.514890Z" }, "scrolled": true }, @@ -1290,7 +1290,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 29, @@ -1318,10 +1318,10 @@ "id": "92683b71", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:55.783914Z", - "iopub.status.busy": "2024-07-25T06:16:55.783825Z", - "iopub.status.idle": "2024-07-25T06:16:55.786436Z", - "shell.execute_reply": "2024-07-25T06:16:55.786196Z" + "iopub.execute_input": "2024-11-05T04:49:38.518723Z", + "iopub.status.busy": "2024-11-05T04:49:38.518598Z", + "iopub.status.idle": "2024-11-05T04:49:38.521529Z", + "shell.execute_reply": "2024-11-05T04:49:38.521217Z" } }, "outputs": [], @@ -1389,10 +1389,10 @@ "id": "6aaa4c89", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:55.788422Z", - "iopub.status.busy": "2024-07-25T06:16:55.788224Z", - "iopub.status.idle": "2024-07-25T06:16:55.790532Z", - "shell.execute_reply": "2024-07-25T06:16:55.790291Z" + "iopub.execute_input": "2024-11-05T04:49:38.524490Z", + "iopub.status.busy": "2024-11-05T04:49:38.524369Z", + "iopub.status.idle": "2024-11-05T04:49:38.526967Z", + "shell.execute_reply": "2024-11-05T04:49:38.526647Z" }, "scrolled": true }, @@ -1416,7 +1416,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 31, @@ -1434,10 +1434,10 @@ "id": "a0bbfe37", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:55.792669Z", - "iopub.status.busy": "2024-07-25T06:16:55.792581Z", - "iopub.status.idle": "2024-07-25T06:16:55.794839Z", - "shell.execute_reply": "2024-07-25T06:16:55.794610Z" + "iopub.execute_input": "2024-11-05T04:49:38.530616Z", + "iopub.status.busy": "2024-11-05T04:49:38.530455Z", + "iopub.status.idle": "2024-11-05T04:49:38.533397Z", + "shell.execute_reply": "2024-11-05T04:49:38.533130Z" }, "scrolled": false }, @@ -1463,7 +1463,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 32, @@ -1500,7 +1500,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0" + "version": "3.10.13" } }, "nbformat": 4, diff --git a/docs/api/S3.md b/docs/api/S3.md index b8f5ab22..d649d130 100644 --- a/docs/api/S3.md +++ b/docs/api/S3.md @@ -31,7 +31,7 @@ For instructions how to use the class, see [Loading and Storing Data](/scaling/d - + @@ -43,7 +43,7 @@ For instructions how to use the class, see [Loading and Storing Data](/scaling/d ## Downloading data - + @@ -60,7 +60,7 @@ For instructions how to use the class, see [Loading and Storing Data](/scaling/d - + @@ -77,7 +77,7 @@ For instructions how to use the class, see [Loading and Storing Data](/scaling/d - + @@ -93,7 +93,7 @@ For instructions how to use the class, see [Loading and Storing Data](/scaling/d - + @@ -110,7 +110,7 @@ For instructions how to use the class, see [Loading and Storing Data](/scaling/d ## Listing objects - + @@ -125,7 +125,7 @@ For instructions how to use the class, see [Loading and Storing Data](/scaling/d - + @@ -142,7 +142,7 @@ For instructions how to use the class, see [Loading and Storing Data](/scaling/d ## Uploading data - + @@ -161,7 +161,7 @@ For instructions how to use the class, see [Loading and Storing Data](/scaling/d - + @@ -177,7 +177,7 @@ For instructions how to use the class, see [Loading and Storing Data](/scaling/d - + @@ -195,7 +195,7 @@ For instructions how to use the class, see [Loading and Storing Data](/scaling/d ## Querying metadata - + @@ -211,7 +211,7 @@ For instructions how to use the class, see [Loading and Storing Data](/scaling/d - + diff --git a/docs/api/argoevent.ipynb b/docs/api/argoevent.ipynb index 7cbff70a..c2bc8618 100644 --- a/docs/api/argoevent.ipynb +++ b/docs/api/argoevent.ipynb @@ -21,10 +21,10 @@ "id": "623be2bc", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.861281Z", - "iopub.status.busy": "2024-07-25T06:16:57.861167Z", - "iopub.status.idle": "2024-07-25T06:16:58.142243Z", - "shell.execute_reply": "2024-07-25T06:16:58.141916Z" + "iopub.execute_input": "2024-11-05T04:49:38.992658Z", + "iopub.status.busy": "2024-11-05T04:49:38.992468Z", + "iopub.status.idle": "2024-11-05T04:49:39.292418Z", + "shell.execute_reply": "2024-11-05T04:49:39.292092Z" } }, "outputs": [], @@ -43,10 +43,10 @@ "id": "1bf3f08e", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:58.144720Z", - "iopub.status.busy": "2024-07-25T06:16:58.144574Z", - "iopub.status.idle": "2024-07-25T06:16:58.150537Z", - "shell.execute_reply": "2024-07-25T06:16:58.150220Z" + "iopub.execute_input": "2024-11-05T04:49:39.295375Z", + "iopub.status.busy": "2024-11-05T04:49:39.295113Z", + "iopub.status.idle": "2024-11-05T04:49:39.301562Z", + "shell.execute_reply": "2024-11-05T04:49:39.301310Z" } }, "outputs": [ @@ -69,7 +69,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -87,10 +87,10 @@ "id": "380eb9ac", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:58.152753Z", - "iopub.status.busy": "2024-07-25T06:16:58.152639Z", - "iopub.status.idle": "2024-07-25T06:16:58.155420Z", - "shell.execute_reply": "2024-07-25T06:16:58.155095Z" + "iopub.execute_input": "2024-11-05T04:49:39.304666Z", + "iopub.status.busy": "2024-11-05T04:49:39.304329Z", + "iopub.status.idle": "2024-11-05T04:49:39.307422Z", + "shell.execute_reply": "2024-11-05T04:49:39.307057Z" } }, "outputs": [ @@ -112,7 +112,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 3, @@ -130,10 +130,10 @@ "id": "31916bae", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:58.157788Z", - "iopub.status.busy": "2024-07-25T06:16:58.157687Z", - "iopub.status.idle": "2024-07-25T06:16:58.160981Z", - "shell.execute_reply": "2024-07-25T06:16:58.160604Z" + "iopub.execute_input": "2024-11-05T04:49:39.310391Z", + "iopub.status.busy": "2024-11-05T04:49:39.310263Z", + "iopub.status.idle": "2024-11-05T04:49:39.313769Z", + "shell.execute_reply": "2024-11-05T04:49:39.313512Z" } }, "outputs": [ @@ -155,7 +155,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 4, @@ -173,10 +173,10 @@ "id": "305678d7", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:58.163921Z", - "iopub.status.busy": "2024-07-25T06:16:58.163779Z", - "iopub.status.idle": "2024-07-25T06:16:58.166454Z", - "shell.execute_reply": "2024-07-25T06:16:58.166148Z" + "iopub.execute_input": "2024-11-05T04:49:39.316992Z", + "iopub.status.busy": "2024-11-05T04:49:39.316882Z", + "iopub.status.idle": "2024-11-05T04:49:39.319433Z", + "shell.execute_reply": "2024-11-05T04:49:39.319145Z" } }, "outputs": [ @@ -198,7 +198,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 5, @@ -235,7 +235,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0" + "version": "3.10.13" } }, "nbformat": 4, diff --git a/docs/api/cards.ipynb b/docs/api/cards.ipynb index 7b032be9..189e773f 100644 --- a/docs/api/cards.ipynb +++ b/docs/api/cards.ipynb @@ -28,10 +28,10 @@ "id": "a5ef9454", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.341488Z", - "iopub.status.busy": "2024-07-25T06:16:56.341371Z", - "iopub.status.idle": "2024-07-25T06:16:56.616466Z", - "shell.execute_reply": "2024-07-25T06:16:56.616152Z" + "iopub.execute_input": "2024-11-05T04:49:36.517129Z", + "iopub.status.busy": "2024-11-05T04:49:36.516791Z", + "iopub.status.idle": "2024-11-05T04:49:36.888477Z", + "shell.execute_reply": "2024-11-05T04:49:36.887469Z" } }, "outputs": [], @@ -63,10 +63,10 @@ "id": "09970e68", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.618794Z", - "iopub.status.busy": "2024-07-25T06:16:56.618638Z", - "iopub.status.idle": "2024-07-25T06:16:56.623639Z", - "shell.execute_reply": "2024-07-25T06:16:56.623386Z" + "iopub.execute_input": "2024-11-05T04:49:36.895231Z", + "iopub.status.busy": "2024-11-05T04:49:36.894879Z", + "iopub.status.idle": "2024-11-05T04:49:36.904303Z", + "shell.execute_reply": "2024-11-05T04:49:36.903410Z" } }, "outputs": [ @@ -93,7 +93,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -111,10 +111,10 @@ "id": "c65fa811", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.625620Z", - "iopub.status.busy": "2024-07-25T06:16:56.625534Z", - "iopub.status.idle": "2024-07-25T06:16:56.629835Z", - "shell.execute_reply": "2024-07-25T06:16:56.629594Z" + "iopub.execute_input": "2024-11-05T04:49:36.908988Z", + "iopub.status.busy": "2024-11-05T04:49:36.908658Z", + "iopub.status.idle": "2024-11-05T04:49:36.916257Z", + "shell.execute_reply": "2024-11-05T04:49:36.915158Z" } }, "outputs": [ @@ -133,7 +133,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 3, @@ -151,10 +151,10 @@ "id": "de91c8c9", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.631875Z", - "iopub.status.busy": "2024-07-25T06:16:56.631777Z", - "iopub.status.idle": "2024-07-25T06:16:56.635751Z", - "shell.execute_reply": "2024-07-25T06:16:56.635509Z" + "iopub.execute_input": "2024-11-05T04:49:36.921493Z", + "iopub.status.busy": "2024-11-05T04:49:36.921141Z", + "iopub.status.idle": "2024-11-05T04:49:36.929003Z", + "shell.execute_reply": "2024-11-05T04:49:36.928222Z" } }, "outputs": [ @@ -173,7 +173,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 4, @@ -191,10 +191,10 @@ "id": "6d76de09", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.637758Z", - "iopub.status.busy": "2024-07-25T06:16:56.637685Z", - "iopub.status.idle": "2024-07-25T06:16:56.640359Z", - "shell.execute_reply": "2024-07-25T06:16:56.640090Z" + "iopub.execute_input": "2024-11-05T04:49:36.936469Z", + "iopub.status.busy": "2024-11-05T04:49:36.936079Z", + "iopub.status.idle": "2024-11-05T04:49:36.941944Z", + "shell.execute_reply": "2024-11-05T04:49:36.941031Z" } }, "outputs": [ @@ -215,7 +215,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 5, @@ -233,10 +233,10 @@ "id": "69f97d00", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.642585Z", - "iopub.status.busy": "2024-07-25T06:16:56.642501Z", - "iopub.status.idle": "2024-07-25T06:16:56.644862Z", - "shell.execute_reply": "2024-07-25T06:16:56.644648Z" + "iopub.execute_input": "2024-11-05T04:49:36.945574Z", + "iopub.status.busy": "2024-11-05T04:49:36.945338Z", + "iopub.status.idle": "2024-11-05T04:49:36.949561Z", + "shell.execute_reply": "2024-11-05T04:49:36.948932Z" } }, "outputs": [ @@ -255,7 +255,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 6, @@ -285,10 +285,10 @@ "id": "53a776e9", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.647190Z", - "iopub.status.busy": "2024-07-25T06:16:56.647106Z", - "iopub.status.idle": "2024-07-25T06:16:56.648657Z", - "shell.execute_reply": "2024-07-25T06:16:56.648403Z" + "iopub.execute_input": "2024-11-05T04:49:36.954572Z", + "iopub.status.busy": "2024-11-05T04:49:36.954397Z", + "iopub.status.idle": "2024-11-05T04:49:36.956818Z", + "shell.execute_reply": "2024-11-05T04:49:36.956237Z" } }, "outputs": [], @@ -312,10 +312,10 @@ "id": "752c4a43", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.650644Z", - "iopub.status.busy": "2024-07-25T06:16:56.650537Z", - "iopub.status.idle": "2024-07-25T06:16:56.657385Z", - "shell.execute_reply": "2024-07-25T06:16:56.657142Z" + "iopub.execute_input": "2024-11-05T04:49:36.960667Z", + "iopub.status.busy": "2024-11-05T04:49:36.960431Z", + "iopub.status.idle": "2024-11-05T04:49:36.968860Z", + "shell.execute_reply": "2024-11-05T04:49:36.968055Z" }, "scrolled": true }, @@ -337,7 +337,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 8, @@ -355,10 +355,10 @@ "id": "d9210681", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.659459Z", - "iopub.status.busy": "2024-07-25T06:16:56.659374Z", - "iopub.status.idle": "2024-07-25T06:16:56.661631Z", - "shell.execute_reply": "2024-07-25T06:16:56.661423Z" + "iopub.execute_input": "2024-11-05T04:49:36.972624Z", + "iopub.status.busy": "2024-11-05T04:49:36.972307Z", + "iopub.status.idle": "2024-11-05T04:49:36.976132Z", + "shell.execute_reply": "2024-11-05T04:49:36.975689Z" } }, "outputs": [ @@ -377,7 +377,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 9, @@ -403,10 +403,10 @@ "id": "9c198774", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.663827Z", - "iopub.status.busy": "2024-07-25T06:16:56.663744Z", - "iopub.status.idle": "2024-07-25T06:16:56.670992Z", - "shell.execute_reply": "2024-07-25T06:16:56.670722Z" + "iopub.execute_input": "2024-11-05T04:49:36.979639Z", + "iopub.status.busy": "2024-11-05T04:49:36.979459Z", + "iopub.status.idle": "2024-11-05T04:49:36.990464Z", + "shell.execute_reply": "2024-11-05T04:49:36.989518Z" } }, "outputs": [ @@ -431,7 +431,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 10, @@ -449,10 +449,10 @@ "id": "da3999a9", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.672822Z", - "iopub.status.busy": "2024-07-25T06:16:56.672729Z", - "iopub.status.idle": "2024-07-25T06:16:56.675433Z", - "shell.execute_reply": "2024-07-25T06:16:56.675083Z" + "iopub.execute_input": "2024-11-05T04:49:36.994962Z", + "iopub.status.busy": "2024-11-05T04:49:36.994787Z", + "iopub.status.idle": "2024-11-05T04:49:37.000486Z", + "shell.execute_reply": "2024-11-05T04:49:36.999281Z" }, "scrolled": true }, @@ -475,7 +475,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 11, @@ -493,10 +493,10 @@ "id": "432ec2c6", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.677654Z", - "iopub.status.busy": "2024-07-25T06:16:56.677554Z", - "iopub.status.idle": "2024-07-25T06:16:56.680038Z", - "shell.execute_reply": "2024-07-25T06:16:56.679807Z" + "iopub.execute_input": "2024-11-05T04:49:37.005117Z", + "iopub.status.busy": "2024-11-05T04:49:37.004931Z", + "iopub.status.idle": "2024-11-05T04:49:37.008765Z", + "shell.execute_reply": "2024-11-05T04:49:37.008325Z" }, "scrolled": true }, @@ -519,7 +519,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 12, @@ -545,10 +545,10 @@ "id": "da4cf95b", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.682447Z", - "iopub.status.busy": "2024-07-25T06:16:56.682350Z", - "iopub.status.idle": "2024-07-25T06:16:56.687254Z", - "shell.execute_reply": "2024-07-25T06:16:56.687007Z" + "iopub.execute_input": "2024-11-05T04:49:37.012865Z", + "iopub.status.busy": "2024-11-05T04:49:37.012684Z", + "iopub.status.idle": "2024-11-05T04:49:37.021612Z", + "shell.execute_reply": "2024-11-05T04:49:37.020275Z" } }, "outputs": [ @@ -571,7 +571,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 13, @@ -597,10 +597,10 @@ "id": "7d1a016c", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.689138Z", - "iopub.status.busy": "2024-07-25T06:16:56.689046Z", - "iopub.status.idle": "2024-07-25T06:16:56.694114Z", - "shell.execute_reply": "2024-07-25T06:16:56.693895Z" + "iopub.execute_input": "2024-11-05T04:49:37.028867Z", + "iopub.status.busy": "2024-11-05T04:49:37.027742Z", + "iopub.status.idle": "2024-11-05T04:49:37.038976Z", + "shell.execute_reply": "2024-11-05T04:49:37.038578Z" } }, "outputs": [ @@ -625,7 +625,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 14, @@ -643,10 +643,10 @@ "id": "1e65fe26", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.695893Z", - "iopub.status.busy": "2024-07-25T06:16:56.695810Z", - "iopub.status.idle": "2024-07-25T06:16:56.698120Z", - "shell.execute_reply": "2024-07-25T06:16:56.697905Z" + "iopub.execute_input": "2024-11-05T04:49:37.042658Z", + "iopub.status.busy": "2024-11-05T04:49:37.042464Z", + "iopub.status.idle": "2024-11-05T04:49:37.047095Z", + "shell.execute_reply": "2024-11-05T04:49:37.046244Z" } }, "outputs": [ @@ -668,7 +668,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 15, @@ -694,10 +694,10 @@ "id": "95806342", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.699962Z", - "iopub.status.busy": "2024-07-25T06:16:56.699869Z", - "iopub.status.idle": "2024-07-25T06:16:56.709072Z", - "shell.execute_reply": "2024-07-25T06:16:56.708826Z" + "iopub.execute_input": "2024-11-05T04:49:37.051905Z", + "iopub.status.busy": "2024-11-05T04:49:37.051614Z", + "iopub.status.idle": "2024-11-05T04:49:37.064954Z", + "shell.execute_reply": "2024-11-05T04:49:37.063951Z" } }, "outputs": [ @@ -716,7 +716,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 16, @@ -734,10 +734,10 @@ "id": "b852d84f", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.710961Z", - "iopub.status.busy": "2024-07-25T06:16:56.710867Z", - "iopub.status.idle": "2024-07-25T06:16:56.713072Z", - "shell.execute_reply": "2024-07-25T06:16:56.712826Z" + "iopub.execute_input": "2024-11-05T04:49:37.069053Z", + "iopub.status.busy": "2024-11-05T04:49:37.068823Z", + "iopub.status.idle": "2024-11-05T04:49:37.073667Z", + "shell.execute_reply": "2024-11-05T04:49:37.072509Z" } }, "outputs": [ @@ -756,7 +756,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 17, @@ -774,10 +774,10 @@ "id": "1636aa90", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.715186Z", - "iopub.status.busy": "2024-07-25T06:16:56.715097Z", - "iopub.status.idle": "2024-07-25T06:16:56.717335Z", - "shell.execute_reply": "2024-07-25T06:16:56.717085Z" + "iopub.execute_input": "2024-11-05T04:49:37.078942Z", + "iopub.status.busy": "2024-11-05T04:49:37.078694Z", + "iopub.status.idle": "2024-11-05T04:49:37.082644Z", + "shell.execute_reply": "2024-11-05T04:49:37.082118Z" } }, "outputs": [ @@ -798,7 +798,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 18, @@ -824,10 +824,10 @@ "id": "5484e8cc", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.719348Z", - "iopub.status.busy": "2024-07-25T06:16:56.719253Z", - "iopub.status.idle": "2024-07-25T06:16:56.725197Z", - "shell.execute_reply": "2024-07-25T06:16:56.724929Z" + "iopub.execute_input": "2024-11-05T04:49:37.086936Z", + "iopub.status.busy": "2024-11-05T04:49:37.086396Z", + "iopub.status.idle": "2024-11-05T04:49:37.100349Z", + "shell.execute_reply": "2024-11-05T04:49:37.099189Z" } }, "outputs": [ @@ -835,19 +835,19 @@ "data": { "text/html": [ "\n", - "

class ProgressBar (max: int = 100, label: str = None, value: int = 0, unit: str = None, metadata: str = None)[source]

metaflow.cards

A Progress bar for tracking progress of any task.

Example:
```
progress_bar = ProgressBar(
    max=100,
    label=\"Progress Bar\",
    value=0,
    unit=\"%\",
    metadata=\"0.1 items/s\"
)
current.card.append(
    progress_bar
)
for i in range(100):
    progress_bar.update(i, metadata=\"%s items/s\" % i)

```

Parameters
----------
max : int
    The maximum value of the progress bar.
label : str, optional
    Optional label for the progress bar.
value : int, optional
    Optional initial value of the progress bar.
unit : str, optional
    Optional unit for the progress bar.
metadata : str, optional
    Optional additional information to show on the progress bar.

\n", + "

class ProgressBar (max: int = 100, label: Optional[str] = None, value: int = 0, unit: Optional[str] = None, metadata: Optional[str] = None)[source]

metaflow.cards

A Progress bar for tracking progress of any task.

Example:
```
progress_bar = ProgressBar(
    max=100,
    label=\"Progress Bar\",
    value=0,
    unit=\"%\",
    metadata=\"0.1 items/s\"
)
current.card.append(
    progress_bar
)
for i in range(100):
    progress_bar.update(i, metadata=\"%s items/s\" % i)

```

Parameters
----------
max : int, default 100
    The maximum value of the progress bar.
label : str, optional, default None
    Optional label for the progress bar.
value : int, default 0
    Optional initial value of the progress bar.
unit : str, optional, default None
    Optional unit for the progress bar.
metadata : str, optional, default None
    Optional additional information to show on the progress bar.

\n", "\n", "\n", "\n", - "\n", + "\n", "\n", "\n", "\n", - "\t\n", - "\t\n", - "\t\n", - "\t\n", - "\t\n", + "\t\n", + "\t\n", + "\t\n", + "\t\n", + "\t\n", "\n", "\n", "\t\n", @@ -855,7 +855,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 19, @@ -873,10 +873,10 @@ "id": "7d6e7dab", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.727258Z", - "iopub.status.busy": "2024-07-25T06:16:56.727172Z", - "iopub.status.idle": "2024-07-25T06:16:56.729479Z", - "shell.execute_reply": "2024-07-25T06:16:56.729197Z" + "iopub.execute_input": "2024-11-05T04:49:37.105430Z", + "iopub.status.busy": "2024-11-05T04:49:37.104965Z", + "iopub.status.idle": "2024-11-05T04:49:37.111399Z", + "shell.execute_reply": "2024-11-05T04:49:37.110634Z" } }, "outputs": [ @@ -884,18 +884,18 @@ "data": { "text/html": [ "\n", - "

method ProgressBar.update (self, new_value: int, metadata: str = None)[source]

#FIXME document

\n", + "

method ProgressBar.update (self, new_value: int, metadata: Optional[str] = None)[source]

#FIXME document

\n", "\n", "\n", "\n", - "\n", + "\n", "\n", "\n", "\n", "" ], "text/plain": [ - "" + "" ] }, "execution_count": 20, @@ -925,10 +925,10 @@ "id": "3dc4881f", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.731605Z", - "iopub.status.busy": "2024-07-25T06:16:56.731524Z", - "iopub.status.idle": "2024-07-25T06:16:56.732959Z", - "shell.execute_reply": "2024-07-25T06:16:56.732740Z" + "iopub.execute_input": "2024-11-05T04:49:37.115876Z", + "iopub.status.busy": "2024-11-05T04:49:37.115346Z", + "iopub.status.idle": "2024-11-05T04:49:37.118844Z", + "shell.execute_reply": "2024-11-05T04:49:37.117826Z" } }, "outputs": [], @@ -943,10 +943,10 @@ "id": "6210f683", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.734788Z", - "iopub.status.busy": "2024-07-25T06:16:56.734707Z", - "iopub.status.idle": "2024-07-25T06:16:56.737988Z", - "shell.execute_reply": "2024-07-25T06:16:56.737749Z" + "iopub.execute_input": "2024-11-05T04:49:37.125369Z", + "iopub.status.busy": "2024-11-05T04:49:37.125177Z", + "iopub.status.idle": "2024-11-05T04:49:37.133133Z", + "shell.execute_reply": "2024-11-05T04:49:37.132037Z" } }, "outputs": [ @@ -970,7 +970,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 22, @@ -988,10 +988,10 @@ "id": "c71b83c5", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.739690Z", - "iopub.status.busy": "2024-07-25T06:16:56.739624Z", - "iopub.status.idle": "2024-07-25T06:16:56.818932Z", - "shell.execute_reply": "2024-07-25T06:16:56.818574Z" + "iopub.execute_input": "2024-11-05T04:49:37.138866Z", + "iopub.status.busy": "2024-11-05T04:49:37.138593Z", + "iopub.status.idle": "2024-11-05T04:49:37.252186Z", + "shell.execute_reply": "2024-11-05T04:49:37.251393Z" } }, "outputs": [ @@ -1002,19 +1002,19 @@ "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", - "File \u001b[0;32m~/mambaforge/envs/docs/lib/python3.11/site-packages/IPython/core/formatters.py:344\u001b[0m, in \u001b[0;36mBaseFormatter.__call__\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m 342\u001b[0m method \u001b[38;5;241m=\u001b[39m get_real_method(obj, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprint_method)\n\u001b[1;32m 343\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m method \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 344\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mmethod\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 345\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 346\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", - "File \u001b[0;32m~/mambaforge/envs/docs/lib/python3.11/site-packages/nbdoc/showdoc.py:218\u001b[0m, in \u001b[0;36mShowDoc._repr_html_\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 216\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_repr_html_\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 217\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mThis method controls what is displayed in Jupyter Notebooks.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m--> 218\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnbhtml\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mjsx\u001b[49m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\n", - "File \u001b[0;32m~/mambaforge/envs/docs/lib/python3.11/site-packages/nbdoc/showdoc.py:253\u001b[0m, in \u001b[0;36mShowDoc.jsx\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 251\u001b[0m nm \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m 252\u001b[0m spoof \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m...\u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdecorator \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mspoofstr\n\u001b[0;32m--> 253\u001b[0m sp \u001b[38;5;241m=\u001b[39m \u001b[43mget_sig_section\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mobj\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mspoofstr\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mspoof\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 254\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mnm\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00msp\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m'\u001b[39m \u001b[38;5;241m+\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnpdocs \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\u001b[39m\u001b[38;5;124m'\u001b[39m\n", - "File \u001b[0;32m~/mambaforge/envs/docs/lib/python3.11/site-packages/nbdoc/showdoc.py:115\u001b[0m, in \u001b[0;36mget_sig_section\u001b[0;34m(obj, spoofstr)\u001b[0m\n\u001b[1;32m 113\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m 114\u001b[0m params \u001b[38;5;241m=\u001b[39m sig\u001b[38;5;241m.\u001b[39mparameters\u001b[38;5;241m.\u001b[39mitems()\n\u001b[0;32m--> 115\u001b[0m jsx_params \u001b[38;5;241m=\u001b[39m \u001b[43m[\u001b[49m\u001b[43mfmt_sig_param\u001b[49m\u001b[43m(\u001b[49m\u001b[43mp\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43m_\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mp\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mparams\u001b[49m\u001b[43m]\u001b[49m\n\u001b[1;32m 116\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 117\u001b[0m jsx_params \u001b[38;5;241m=\u001b[39m [\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\u001b[39m\u001b[38;5;124m'\u001b[39m]\n", - "File \u001b[0;32m~/mambaforge/envs/docs/lib/python3.11/site-packages/nbdoc/showdoc.py:115\u001b[0m, in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 113\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m 114\u001b[0m params \u001b[38;5;241m=\u001b[39m sig\u001b[38;5;241m.\u001b[39mparameters\u001b[38;5;241m.\u001b[39mitems()\n\u001b[0;32m--> 115\u001b[0m jsx_params \u001b[38;5;241m=\u001b[39m [\u001b[43mfmt_sig_param\u001b[49m\u001b[43m(\u001b[49m\u001b[43mp\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m _, p \u001b[38;5;129;01min\u001b[39;00m params]\n\u001b[1;32m 116\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 117\u001b[0m jsx_params \u001b[38;5;241m=\u001b[39m [\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\u001b[39m\u001b[38;5;124m'\u001b[39m]\n", - "File \u001b[0;32m~/mambaforge/envs/docs/lib/python3.11/site-packages/nbdoc/showdoc.py:99\u001b[0m, in \u001b[0;36mfmt_sig_param\u001b[0;34m(p)\u001b[0m\n\u001b[1;32m 96\u001b[0m prefix \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m 99\u001b[0m prefix \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtype=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[43mp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mannotation\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;18;43m__name__\u001b[39;49m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m 100\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m p\u001b[38;5;241m.\u001b[39mdefault \u001b[38;5;241m!=\u001b[39m inspect\u001b[38;5;241m.\u001b[39m_empty:\n\u001b[1;32m 101\u001b[0m prefix \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdefault=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mp\u001b[38;5;241m.\u001b[39mdefault\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m'\u001b[39m\n", + "File \u001b[0;32m~/mambaforge/envs/metaflow-oss-docs/lib/python3.10/site-packages/IPython/core/formatters.py:344\u001b[0m, in \u001b[0;36mBaseFormatter.__call__\u001b[0;34m(self, obj)\u001b[0m\n\u001b[1;32m 342\u001b[0m method \u001b[38;5;241m=\u001b[39m get_real_method(obj, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprint_method)\n\u001b[1;32m 343\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m method \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 344\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mmethod\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 345\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 346\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n", + "File \u001b[0;32m~/mambaforge/envs/metaflow-oss-docs/lib/python3.10/site-packages/nbdoc/showdoc.py:218\u001b[0m, in \u001b[0;36mShowDoc._repr_html_\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 216\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_repr_html_\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m 217\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mThis method controls what is displayed in Jupyter Notebooks.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m--> 218\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnbhtml\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mjsx\u001b[49m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\n", + "File \u001b[0;32m~/mambaforge/envs/metaflow-oss-docs/lib/python3.10/site-packages/nbdoc/showdoc.py:253\u001b[0m, in \u001b[0;36mShowDoc.jsx\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 251\u001b[0m nm \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m 252\u001b[0m spoof \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m...\u001b[39m\u001b[38;5;124m'\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdecorator \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mspoofstr\n\u001b[0;32m--> 253\u001b[0m sp \u001b[38;5;241m=\u001b[39m \u001b[43mget_sig_section\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mobj\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mspoofstr\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mspoof\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 254\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mnm\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00msp\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m'\u001b[39m \u001b[38;5;241m+\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnpdocs \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\u001b[39m\u001b[38;5;124m'\u001b[39m\n", + "File \u001b[0;32m~/mambaforge/envs/metaflow-oss-docs/lib/python3.10/site-packages/nbdoc/showdoc.py:115\u001b[0m, in \u001b[0;36mget_sig_section\u001b[0;34m(obj, spoofstr)\u001b[0m\n\u001b[1;32m 113\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m 114\u001b[0m params \u001b[38;5;241m=\u001b[39m sig\u001b[38;5;241m.\u001b[39mparameters\u001b[38;5;241m.\u001b[39mitems()\n\u001b[0;32m--> 115\u001b[0m jsx_params \u001b[38;5;241m=\u001b[39m [fmt_sig_param(p) \u001b[38;5;28;01mfor\u001b[39;00m _, p \u001b[38;5;129;01min\u001b[39;00m params]\n\u001b[1;32m 116\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 117\u001b[0m jsx_params \u001b[38;5;241m=\u001b[39m [\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\u001b[39m\u001b[38;5;124m'\u001b[39m]\n", + "File \u001b[0;32m~/mambaforge/envs/metaflow-oss-docs/lib/python3.10/site-packages/nbdoc/showdoc.py:115\u001b[0m, in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 113\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m 114\u001b[0m params \u001b[38;5;241m=\u001b[39m sig\u001b[38;5;241m.\u001b[39mparameters\u001b[38;5;241m.\u001b[39mitems()\n\u001b[0;32m--> 115\u001b[0m jsx_params \u001b[38;5;241m=\u001b[39m [\u001b[43mfmt_sig_param\u001b[49m\u001b[43m(\u001b[49m\u001b[43mp\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m _, p \u001b[38;5;129;01min\u001b[39;00m params]\n\u001b[1;32m 116\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 117\u001b[0m jsx_params \u001b[38;5;241m=\u001b[39m [\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\u001b[39m\u001b[38;5;124m'\u001b[39m]\n", + "File \u001b[0;32m~/mambaforge/envs/metaflow-oss-docs/lib/python3.10/site-packages/nbdoc/showdoc.py:99\u001b[0m, in \u001b[0;36mfmt_sig_param\u001b[0;34m(p)\u001b[0m\n\u001b[1;32m 96\u001b[0m prefix \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m 99\u001b[0m prefix \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtype=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[43mp\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mannotation\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;18;43m__name__\u001b[39;49m\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m 100\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m p\u001b[38;5;241m.\u001b[39mdefault \u001b[38;5;241m!=\u001b[39m inspect\u001b[38;5;241m.\u001b[39m_empty:\n\u001b[1;32m 101\u001b[0m prefix \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdefault=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mp\u001b[38;5;241m.\u001b[39mdefault\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m \u001b[39m\u001b[38;5;124m'\u001b[39m\n", "\u001b[0;31mAttributeError\u001b[0m: 'str' object has no attribute '__name__'" ] }, { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 23, @@ -1032,10 +1032,10 @@ "id": "e2c444ee", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.821647Z", - "iopub.status.busy": "2024-07-25T06:16:56.821534Z", - "iopub.status.idle": "2024-07-25T06:16:56.824138Z", - "shell.execute_reply": "2024-07-25T06:16:56.823867Z" + "iopub.execute_input": "2024-11-05T04:49:37.257347Z", + "iopub.status.busy": "2024-11-05T04:49:37.257026Z", + "iopub.status.idle": "2024-11-05T04:49:37.265500Z", + "shell.execute_reply": "2024-11-05T04:49:37.263319Z" } }, "outputs": [ @@ -1054,7 +1054,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 24, @@ -1072,10 +1072,10 @@ "id": "e360ce50", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.826778Z", - "iopub.status.busy": "2024-07-25T06:16:56.826668Z", - "iopub.status.idle": "2024-07-25T06:16:56.829343Z", - "shell.execute_reply": "2024-07-25T06:16:56.829068Z" + "iopub.execute_input": "2024-11-05T04:49:37.271958Z", + "iopub.status.busy": "2024-11-05T04:49:37.271702Z", + "iopub.status.idle": "2024-11-05T04:49:37.277913Z", + "shell.execute_reply": "2024-11-05T04:49:37.277125Z" } }, "outputs": [ @@ -1094,7 +1094,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 25, @@ -1131,7 +1131,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0" + "version": "3.10.13" } }, "nbformat": 4, diff --git a/docs/api/cards.md b/docs/api/cards.md index ca60828e..056d156d 100644 --- a/docs/api/cards.md +++ b/docs/api/cards.md @@ -236,15 +236,15 @@ The components are added to cards in `@step` methods (or functions called from s - + - - - - - + + + + + @@ -255,7 +255,7 @@ The components are added to cards in `@step` methods (or functions called from s - + @@ -295,7 +295,7 @@ ShowDoc(MetaflowCard.render) AttributeError Traceback (most recent call last) - File ~/mambaforge/envs/docs/lib/python3.11/site-packages/IPython/core/formatters.py:344, in BaseFormatter.__call__(self, obj) + File ~/mambaforge/envs/metaflow-oss-docs/lib/python3.10/site-packages/IPython/core/formatters.py:344, in BaseFormatter.__call__(self, obj) 342 method = get_real_method(obj, self.print_method) 343 if method is not None: --> 344 return method() @@ -303,20 +303,20 @@ ShowDoc(MetaflowCard.render) 346 else: - File ~/mambaforge/envs/docs/lib/python3.11/site-packages/nbdoc/showdoc.py:218, in ShowDoc._repr_html_(self) + File ~/mambaforge/envs/metaflow-oss-docs/lib/python3.10/site-packages/nbdoc/showdoc.py:218, in ShowDoc._repr_html_(self) 216 def _repr_html_(self): 217 "This method controls what is displayed in Jupyter Notebooks." --> 218 return f'\n{self.nbhtml}\n\n{self.jsx}' - File ~/mambaforge/envs/docs/lib/python3.11/site-packages/nbdoc/showdoc.py:253, in ShowDoc.jsx(self) + File ~/mambaforge/envs/metaflow-oss-docs/lib/python3.10/site-packages/nbdoc/showdoc.py:253, in ShowDoc.jsx(self) 251 nm = f'' 252 spoof = '...' if self.decorator else self.spoofstr --> 253 sp = get_sig_section(self.obj, spoofstr=spoof) 254 return f'{nm}\n{sp}\n' + self.npdocs + '\n' - File ~/mambaforge/envs/docs/lib/python3.11/site-packages/nbdoc/showdoc.py:115, in get_sig_section(obj, spoofstr) + File ~/mambaforge/envs/metaflow-oss-docs/lib/python3.10/site-packages/nbdoc/showdoc.py:115, in get_sig_section(obj, spoofstr) 113 return '' 114 params = sig.parameters.items() --> 115 jsx_params = [fmt_sig_param(p) for _, p in params] @@ -324,7 +324,7 @@ ShowDoc(MetaflowCard.render) 117 jsx_params = [f''] - File ~/mambaforge/envs/docs/lib/python3.11/site-packages/nbdoc/showdoc.py:115, in (.0) + File ~/mambaforge/envs/metaflow-oss-docs/lib/python3.10/site-packages/nbdoc/showdoc.py:115, in (.0) 113 return '' 114 params = sig.parameters.items() --> 115 jsx_params = [fmt_sig_param(p) for _, p in params] @@ -332,7 +332,7 @@ ShowDoc(MetaflowCard.render) 117 jsx_params = [f''] - File ~/mambaforge/envs/docs/lib/python3.11/site-packages/nbdoc/showdoc.py:99, in fmt_sig_param(p) + File ~/mambaforge/envs/metaflow-oss-docs/lib/python3.10/site-packages/nbdoc/showdoc.py:99, in fmt_sig_param(p) 96 prefix = f' 99 prefix += f'type="{p.annotation.__name__}" ' @@ -346,7 +346,7 @@ ShowDoc(MetaflowCard.render) - + ``` diff --git a/docs/api/client.ipynb b/docs/api/client.ipynb index c0029283..4facc055 100644 --- a/docs/api/client.ipynb +++ b/docs/api/client.ipynb @@ -70,10 +70,10 @@ "id": "d1e0717d-dcdd-43f8-9116-1fbe15200503", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.350381Z", - "iopub.status.busy": "2024-07-25T06:16:57.350292Z", - "iopub.status.idle": "2024-07-25T06:16:57.577789Z", - "shell.execute_reply": "2024-07-25T06:16:57.577486Z" + "iopub.execute_input": "2024-11-05T04:49:37.552230Z", + "iopub.status.busy": "2024-11-05T04:49:37.551927Z", + "iopub.status.idle": "2024-11-05T04:49:37.845484Z", + "shell.execute_reply": "2024-11-05T04:49:37.845055Z" } }, "outputs": [], @@ -108,10 +108,10 @@ "id": "be46136f-68d0-4b73-b53c-a6da21023ea7", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.580144Z", - "iopub.status.busy": "2024-07-25T06:16:57.580007Z", - "iopub.status.idle": "2024-07-25T06:16:57.635667Z", - "shell.execute_reply": "2024-07-25T06:16:57.635201Z" + "iopub.execute_input": "2024-11-05T04:49:37.848529Z", + "iopub.status.busy": "2024-11-05T04:49:37.848358Z", + "iopub.status.idle": "2024-11-05T04:49:37.913642Z", + "shell.execute_reply": "2024-11-05T04:49:37.913253Z" } }, "outputs": [ @@ -119,11 +119,11 @@ "data": { "text/html": [ "\n", - "

class Metaflow ()[source]

Entry point to all objects in the Metaflow universe.

This object can be used to list all the flows present either through the explicit property
or by iterating over this object.

Attributes
----------
flows : List[Flow]
    Returns the list of all `Flow` objects known to this metadata provider. Note that only
    flows present in the current namespace will be returned. A `Flow` is present in a namespace
    if it has at least one run in the namespace.

\n", + "

class Metaflow (_current_metadata: Optional[str] = None)[source]

Entry point to all objects in the Metaflow universe.

This object can be used to list all the flows present either through the explicit property
or by iterating over this object.

Attributes
----------
flows : List[Flow]
    Returns the list of all `Flow` objects known to this metadata provider. Note that only
    flows present in the current namespace will be returned. A `Flow` is present in a namespace
    if it has at least one run in the namespace.

\n", "\n", - "\n", + "\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -132,7 +132,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -160,10 +160,10 @@ "id": "aa03d75d-c0cc-4d5b-a65a-bab43e5061fe", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.639238Z", - "iopub.status.busy": "2024-07-25T06:16:57.639123Z", - "iopub.status.idle": "2024-07-25T06:16:57.649624Z", - "shell.execute_reply": "2024-07-25T06:16:57.649377Z" + "iopub.execute_input": "2024-11-05T04:49:37.916463Z", + "iopub.status.busy": "2024-11-05T04:49:37.916330Z", + "iopub.status.idle": "2024-11-05T04:49:37.929240Z", + "shell.execute_reply": "2024-11-05T04:49:37.928875Z" } }, "outputs": [ @@ -171,9 +171,9 @@ "data": { "text/html": [ "\n", - "

class Flow pathspec[source]

A Flow represents all existing flows with a certain name, in other words,
classes derived from `FlowSpec`. A container of `Run` objects.

Attributes
----------
latest_run : Run
    Latest `Run` (in progress or completed, successfully or not) of this flow.
latest_successful_run : Run
    Latest successfully completed `Run` of this flow.

\n", + "

class Flow pathspec[source]

A Flow represents all existing flows with a certain name, in other words,
classes derived from `FlowSpec`. A container of `Run` objects.

Attributes
----------
latest_run : Run
    Latest `Run` (in progress or completed, successfully or not) of this flow.
latest_successful_run : Run
    Latest successfully completed `Run` of this flow.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -185,7 +185,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 3, @@ -203,10 +203,10 @@ "id": "e0da0de1", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.651561Z", - "iopub.status.busy": "2024-07-25T06:16:57.651461Z", - "iopub.status.idle": "2024-07-25T06:16:57.654260Z", - "shell.execute_reply": "2024-07-25T06:16:57.653992Z" + "iopub.execute_input": "2024-11-05T04:49:37.931942Z", + "iopub.status.busy": "2024-11-05T04:49:37.931810Z", + "iopub.status.idle": "2024-11-05T04:49:37.936211Z", + "shell.execute_reply": "2024-11-05T04:49:37.934856Z" } }, "outputs": [ @@ -214,9 +214,9 @@ "data": { "text/html": [ "\n", - "

method Flow.runs (self, *tags: str) -> Iterator[metaflow.client.core.Run][source]

Returns an iterator over all `Run`s of this flow.

An optional filter is available that allows you to filter on tags.
If multiple tags are specified, only runs that have all the
specified tags are returned.

Parameters
----------
tags : str
    Tags to match.

Yields
------
Run
    `Run` objects in this flow.

\n", + "

method Flow.runs (self, *tags: str) -> Iterator[metaflow.client.core.Run][source]

Returns an iterator over all `Run`s of this flow.

An optional filter is available that allows you to filter on tags.
If multiple tags are specified, only runs that have all the
specified tags are returned.

Parameters
----------
tags : str
    Tags to match.

Yields
------
Run
    `Run` objects in this flow.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -230,7 +230,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 4, @@ -256,10 +256,10 @@ "id": "4302ed53", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.656361Z", - "iopub.status.busy": "2024-07-25T06:16:57.656263Z", - "iopub.status.idle": "2024-07-25T06:16:57.667437Z", - "shell.execute_reply": "2024-07-25T06:16:57.667078Z" + "iopub.execute_input": "2024-11-05T04:49:37.939545Z", + "iopub.status.busy": "2024-11-05T04:49:37.939402Z", + "iopub.status.idle": "2024-11-05T04:49:37.952847Z", + "shell.execute_reply": "2024-11-05T04:49:37.952453Z" } }, "outputs": [ @@ -267,9 +267,9 @@ "data": { "text/html": [ "\n", - "

class Run pathspec[source]

A `Run` represents an execution of a `Flow`. It is a container of `Step`s.

Attributes
----------
data : MetaflowData
    a shortcut to run['end'].task.data, i.e. data produced by this run.
successful : bool
    True if the run completed successfully.
finished : bool
    True if the run completed.
finished_at : datetime
    Time this run finished.
code : MetaflowCode
    Code package for this run (if present). See `MetaflowCode`.
trigger : MetaflowTrigger
    Information about event(s) that triggered this run (if present). See `MetaflowTrigger`.
end_task : Task
    `Task` for the end step (if it is present already).

\n", + "

class Run pathspec[source]

A `Run` represents an execution of a `Flow`. It is a container of `Step`s.

Attributes
----------
data : MetaflowData
    a shortcut to run['end'].task.data, i.e. data produced by this run.
successful : bool
    True if the run completed successfully.
finished : bool
    True if the run completed.
finished_at : datetime
    Time this run finished.
code : MetaflowCode
    Code package for this run (if present). See `MetaflowCode`.
trigger : MetaflowTrigger
    Information about event(s) that triggered this run (if present). See `MetaflowTrigger`.
end_task : Task
    `Task` for the end step (if it is present already).

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -286,7 +286,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 5, @@ -304,10 +304,10 @@ "id": "5d0ad8c7", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.670190Z", - "iopub.status.busy": "2024-07-25T06:16:57.670055Z", - "iopub.status.idle": "2024-07-25T06:16:57.672698Z", - "shell.execute_reply": "2024-07-25T06:16:57.672414Z" + "iopub.execute_input": "2024-11-05T04:49:37.955670Z", + "iopub.status.busy": "2024-11-05T04:49:37.955534Z", + "iopub.status.idle": "2024-11-05T04:49:37.958109Z", + "shell.execute_reply": "2024-11-05T04:49:37.957869Z" } }, "outputs": [ @@ -315,9 +315,9 @@ "data": { "text/html": [ "\n", - "

method Run.add_tag (self, tag: str)[source]

Add a tag to this `Run`.

Note that if the tag is already a system tag, it is not added as a user tag,
and no error is thrown.

Parameters
----------
tag : str
    Tag to add.

\n", + "

method Run.add_tag (self, tag: str)[source]

Add a tag to this `Run`.

Note that if the tag is already a system tag, it is not added as a user tag,
and no error is thrown.

Parameters
----------
tag : str
    Tag to add.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -328,7 +328,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 6, @@ -346,10 +346,10 @@ "id": "037da6e3", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.675043Z", - "iopub.status.busy": "2024-07-25T06:16:57.674938Z", - "iopub.status.idle": "2024-07-25T06:16:57.677501Z", - "shell.execute_reply": "2024-07-25T06:16:57.677260Z" + "iopub.execute_input": "2024-11-05T04:49:37.960757Z", + "iopub.status.busy": "2024-11-05T04:49:37.960643Z", + "iopub.status.idle": "2024-11-05T04:49:37.963280Z", + "shell.execute_reply": "2024-11-05T04:49:37.963021Z" } }, "outputs": [ @@ -357,9 +357,9 @@ "data": { "text/html": [ "\n", - "

method Run.add_tags (self, tags: Iterable[str])[source]

Add one or more tags to this `Run`.

Note that if any tag is already a system tag, it is not added as a user tag
and no error is thrown.

Parameters
----------
tags : Iterable[str]
    Tags to add.

\n", + "

method Run.add_tags (self, tags: Iterable[str])[source]

Add one or more tags to this `Run`.

Note that if any tag is already a system tag, it is not added as a user tag
and no error is thrown.

Parameters
----------
tags : Iterable[str]
    Tags to add.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -370,7 +370,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 7, @@ -388,10 +388,10 @@ "id": "0ebd8693", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.679621Z", - "iopub.status.busy": "2024-07-25T06:16:57.679535Z", - "iopub.status.idle": "2024-07-25T06:16:57.682013Z", - "shell.execute_reply": "2024-07-25T06:16:57.681753Z" + "iopub.execute_input": "2024-11-05T04:49:37.966056Z", + "iopub.status.busy": "2024-11-05T04:49:37.965949Z", + "iopub.status.idle": "2024-11-05T04:49:37.968435Z", + "shell.execute_reply": "2024-11-05T04:49:37.968186Z" } }, "outputs": [ @@ -399,9 +399,9 @@ "data": { "text/html": [ "\n", - "

method Run.remove_tag (self, tag: str)[source]

Remove one tag from this `Run`.

Removing a system tag is an error. Removing a non-existent
user tag is a no-op.

Parameters
----------
tag : str
    Tag to remove.

\n", + "

method Run.remove_tag (self, tag: str)[source]

Remove one tag from this `Run`.

Removing a system tag is an error. Removing a non-existent
user tag is a no-op.

Parameters
----------
tag : str
    Tag to remove.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -412,7 +412,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 8, @@ -430,10 +430,10 @@ "id": "0dd3c17a", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.683864Z", - "iopub.status.busy": "2024-07-25T06:16:57.683778Z", - "iopub.status.idle": "2024-07-25T06:16:57.686125Z", - "shell.execute_reply": "2024-07-25T06:16:57.685886Z" + "iopub.execute_input": "2024-11-05T04:49:37.971204Z", + "iopub.status.busy": "2024-11-05T04:49:37.971056Z", + "iopub.status.idle": "2024-11-05T04:49:37.973850Z", + "shell.execute_reply": "2024-11-05T04:49:37.973461Z" } }, "outputs": [ @@ -441,9 +441,9 @@ "data": { "text/html": [ "\n", - "

method Run.remove_tags (self, tags: Iterable[str])[source]

Remove one or more tags to this `Run`.

Removing a system tag will result in an error. Removing a non-existent
user tag is a no-op.

Parameters
----------
tags : Iterable[str]
    Tags to remove.

\n", + "

method Run.remove_tags (self, tags: Iterable[str])[source]

Remove one or more tags to this `Run`.

Removing a system tag will result in an error. Removing a non-existent
user tag is a no-op.

Parameters
----------
tags : Iterable[str]
    Tags to remove.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -454,7 +454,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 9, @@ -472,10 +472,10 @@ "id": "2e97ed7e", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.688756Z", - "iopub.status.busy": "2024-07-25T06:16:57.688669Z", - "iopub.status.idle": "2024-07-25T06:16:57.691195Z", - "shell.execute_reply": "2024-07-25T06:16:57.690885Z" + "iopub.execute_input": "2024-11-05T04:49:37.976785Z", + "iopub.status.busy": "2024-11-05T04:49:37.976630Z", + "iopub.status.idle": "2024-11-05T04:49:37.979681Z", + "shell.execute_reply": "2024-11-05T04:49:37.979374Z" } }, "outputs": [ @@ -483,9 +483,9 @@ "data": { "text/html": [ "\n", - "

method Run.replace_tag (self, tag_to_remove: str, tag_to_add: str)[source]

Remove a tag and add a tag atomically. Removal is done first.
The rules for `Run.add_tag` and `Run.remove_tag` also apply here.

Parameters
----------
tag_to_remove : str
    Tag to remove.
tag_to_add : str
    Tag to add.

\n", + "

method Run.replace_tag (self, tag_to_remove: str, tag_to_add: str)[source]

Remove a tag and add a tag atomically. Removal is done first.
The rules for `Run.add_tag` and `Run.remove_tag` also apply here.

Parameters
----------
tag_to_remove : str
    Tag to remove.
tag_to_add : str
    Tag to add.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -497,7 +497,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 10, @@ -515,10 +515,10 @@ "id": "782ee581", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.693713Z", - "iopub.status.busy": "2024-07-25T06:16:57.693614Z", - "iopub.status.idle": "2024-07-25T06:16:57.696060Z", - "shell.execute_reply": "2024-07-25T06:16:57.695825Z" + "iopub.execute_input": "2024-11-05T04:49:37.983129Z", + "iopub.status.busy": "2024-11-05T04:49:37.982992Z", + "iopub.status.idle": "2024-11-05T04:49:37.986051Z", + "shell.execute_reply": "2024-11-05T04:49:37.985749Z" } }, "outputs": [ @@ -526,9 +526,9 @@ "data": { "text/html": [ "\n", - "

method Run.replace_tags (self, tags_to_remove: Iterable[str], tags_to_add: Iterable[str])[source]

Remove and add tags atomically; the removal is done first.
The rules for `Run.add_tag` and `Run.remove_tag` also apply here.

Parameters
----------
tags_to_remove : Iterable[str]
    Tags to remove.
tags_to_add : Iterable[str]
    Tags to add.

\n", + "

method Run.replace_tags (self, tags_to_remove: Iterable[str], tags_to_add: Iterable[str])[source]

Remove and add tags atomically; the removal is done first.
The rules for `Run.add_tag` and `Run.remove_tag` also apply here.

Parameters
----------
tags_to_remove : Iterable[str]
    Tags to remove.
tags_to_add : Iterable[str]
    Tags to add.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -540,7 +540,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 11, @@ -566,10 +566,10 @@ "id": "e56c7d2d", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.698624Z", - "iopub.status.busy": "2024-07-25T06:16:57.698526Z", - "iopub.status.idle": "2024-07-25T06:16:57.708117Z", - "shell.execute_reply": "2024-07-25T06:16:57.707865Z" + "iopub.execute_input": "2024-11-05T04:49:37.988507Z", + "iopub.status.busy": "2024-11-05T04:49:37.988415Z", + "iopub.status.idle": "2024-11-05T04:49:38.016804Z", + "shell.execute_reply": "2024-11-05T04:49:38.015973Z" } }, "outputs": [ @@ -577,9 +577,9 @@ "data": { "text/html": [ "\n", - "

class Step pathspec[source]

A `Step` represents a user-defined step, that is, a method annotated with the `@step` decorator.

It contains `Task` objects associated with the step, that is, all executions of the
`Step`. The step may contain multiple `Task`s in the case of a foreach step.

Attributes
----------
task : Task
    The first `Task` object in this step. This is a shortcut for retrieving the only
    task contained in a non-foreach step.
finished_at : datetime
    Time when the latest `Task` of this step finished. Note that in the case of foreaches,
    this time may change during execution of the step.
environment_info : Dict[str, Any]
    Information about the execution environment.

\n", + "

class Step pathspec[source]

A `Step` represents a user-defined step, that is, a method annotated with the `@step` decorator.

It contains `Task` objects associated with the step, that is, all executions of the
`Step`. The step may contain multiple `Task`s in the case of a foreach step.

Attributes
----------
task : Task
    The first `Task` object in this step. This is a shortcut for retrieving the only
    task contained in a non-foreach step.
finished_at : datetime
    Time when the latest `Task` of this step finished. Note that in the case of foreaches,
    this time may change during execution of the step.
environment_info : Dict[str, Any]
    Information about the execution environment.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -592,7 +592,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 12, @@ -618,10 +618,10 @@ "id": "43db4359", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.710327Z", - "iopub.status.busy": "2024-07-25T06:16:57.710242Z", - "iopub.status.idle": "2024-07-25T06:16:57.721477Z", - "shell.execute_reply": "2024-07-25T06:16:57.721224Z" + "iopub.execute_input": "2024-11-05T04:49:38.033762Z", + "iopub.status.busy": "2024-11-05T04:49:38.032820Z", + "iopub.status.idle": "2024-11-05T04:49:38.047480Z", + "shell.execute_reply": "2024-11-05T04:49:38.047172Z" } }, "outputs": [ @@ -629,9 +629,9 @@ "data": { "text/html": [ "\n", - "

class Task pathspec, attempt=None[source]

A `Task` represents an execution of a `Step`.

It contains all `DataArtifact` objects produced by the task as
well as metadata related to execution.

Note that the `@retry` decorator may cause multiple attempts of
the task to be present. Usually you want the latest attempt, which
is what instantiating a `Task` object returns by default. If
you need to e.g. retrieve logs from a failed attempt, you can
explicitly get information about a specific attempt by using the
following syntax when creating a task:

`Task('flow/run/step/task', attempt=)`

where `attempt=0` corresponds to the first attempt etc.

Attributes
----------
metadata : List[Metadata]
    List of all metadata events associated with the task.
metadata_dict : Dict[str, str]
    A condensed version of `metadata`: A dictionary where keys
    are names of metadata events and values the latest corresponding event.
data : MetaflowData
    Container of all data artifacts produced by this task. Note that this
    call downloads all data locally, so it can be slower than accessing
    artifacts individually. See `MetaflowData` for more information.
artifacts : MetaflowArtifacts
    Container of `DataArtifact` objects produced by this task.
successful : bool
    True if the task completed successfully.
finished : bool
    True if the task completed.
exception : object
    Exception raised by this task if there was one.
finished_at : datetime
    Time this task finished.
runtime_name : str
    Runtime this task was executed on.
stdout : str
    Standard output for the task execution.
stderr : str
    Standard error output for the task execution.
code : MetaflowCode
    Code package for this task (if present). See `MetaflowCode`.
environment_info : Dict[str, str]
    Information about the execution environment.

\n", + "

class Task pathspec, attempt=None[source]

A `Task` represents an execution of a `Step`.

It contains all `DataArtifact` objects produced by the task as
well as metadata related to execution.

Note that the `@retry` decorator may cause multiple attempts of
the task to be present. Usually you want the latest attempt, which
is what instantiating a `Task` object returns by default. If
you need to e.g. retrieve logs from a failed attempt, you can
explicitly get information about a specific attempt by using the
following syntax when creating a task:

`Task('flow/run/step/task', attempt=)`

where `attempt=0` corresponds to the first attempt etc.

Attributes
----------
metadata : List[Metadata]
    List of all metadata events associated with the task.
metadata_dict : Dict[str, str]
    A condensed version of `metadata`: A dictionary where keys
    are names of metadata events and values the latest corresponding event.
data : MetaflowData
    Container of all data artifacts produced by this task. Note that this
    call downloads all data locally, so it can be slower than accessing
    artifacts individually. See `MetaflowData` for more information.
artifacts : MetaflowArtifacts
    Container of `DataArtifact` objects produced by this task.
successful : bool
    True if the task completed successfully.
finished : bool
    True if the task completed.
exception : object
    Exception raised by this task if there was one.
finished_at : datetime
    Time this task finished.
runtime_name : str
    Runtime this task was executed on.
stdout : str
    Standard output for the task execution.
stderr : str
    Standard error output for the task execution.
code : MetaflowCode
    Code package for this task (if present). See `MetaflowCode`.
environment_info : Dict[str, str]
    Information about the execution environment.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -654,7 +654,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 13, @@ -672,10 +672,10 @@ "id": "fdb93b31", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.723708Z", - "iopub.status.busy": "2024-07-25T06:16:57.723614Z", - "iopub.status.idle": "2024-07-25T06:16:57.726284Z", - "shell.execute_reply": "2024-07-25T06:16:57.726052Z" + "iopub.execute_input": "2024-11-05T04:49:38.051002Z", + "iopub.status.busy": "2024-11-05T04:49:38.050823Z", + "iopub.status.idle": "2024-11-05T04:49:38.054388Z", + "shell.execute_reply": "2024-11-05T04:49:38.054015Z" } }, "outputs": [ @@ -683,9 +683,9 @@ "data": { "text/html": [ "\n", - "

method Task.loglines (self, stream: str, as_unicode: bool = True, meta_dict: Optional[Dict[str, Any]] = None) -> Iterator[Tuple[datetime.datetime, str]][source]

Return an iterator over (utc_timestamp, logline) tuples.

Parameters
----------
stream : str
    Either 'stdout' or 'stderr'.
as_unicode : bool, default: True
    If as_unicode=False, each logline is returned as a byte object. Otherwise,
    it is returned as a (unicode) string.

Yields
------
Tuple[datetime, str]
    Tuple of timestamp, logline pairs.

\n", + "

method Task.loglines (self, stream: str, as_unicode: bool = True, meta_dict: Optional[Dict[str, Any]] = None) -> Iterator[Tuple[datetime.datetime, str]][source]

Return an iterator over (utc_timestamp, logline) tuples.

Parameters
----------
stream : str
    Either 'stdout' or 'stderr'.
as_unicode : bool, default: True
    If as_unicode=False, each logline is returned as a byte object. Otherwise,
    it is returned as a (unicode) string.

Yields
------
Tuple[datetime, str]
    Tuple of timestamp, logline pairs.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -700,7 +700,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 14, @@ -726,10 +726,10 @@ "id": "5dad835f", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.728300Z", - "iopub.status.busy": "2024-07-25T06:16:57.728224Z", - "iopub.status.idle": "2024-07-25T06:16:57.756648Z", - "shell.execute_reply": "2024-07-25T06:16:57.756377Z" + "iopub.execute_input": "2024-11-05T04:49:38.057562Z", + "iopub.status.busy": "2024-11-05T04:49:38.057346Z", + "iopub.status.idle": "2024-11-05T04:49:38.067497Z", + "shell.execute_reply": "2024-11-05T04:49:38.067167Z" } }, "outputs": [ @@ -737,9 +737,9 @@ "data": { "text/html": [ "\n", - "

class DataArtifact pathspec[source]

A single data artifact and associated metadata. Note that this object does
not contain other objects as it is the leaf object in the hierarchy.

Attributes
----------
data : object
    The data contained in this artifact, that is, the object produced during
    execution of this run.
sha : string
    A unique ID of this artifact.
finished_at : datetime
    Corresponds roughly to the `Task.finished_at` time of the parent `Task`.
    An alias for `DataArtifact.created_at`.

\n", + "

class DataArtifact pathspec[source]

A single data artifact and associated metadata. Note that this object does
not contain other objects as it is the leaf object in the hierarchy.

Attributes
----------
data : object
    The data contained in this artifact, that is, the object produced during
    execution of this run.
sha : string
    A unique ID of this artifact.
finished_at : datetime
    Corresponds roughly to the `Task.finished_at` time of the parent `Task`.
    An alias for `DataArtifact.created_at`.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -752,7 +752,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 15, @@ -786,10 +786,10 @@ "id": "7c6ef99c", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.759463Z", - "iopub.status.busy": "2024-07-25T06:16:57.759351Z", - "iopub.status.idle": "2024-07-25T06:16:57.767882Z", - "shell.execute_reply": "2024-07-25T06:16:57.767632Z" + "iopub.execute_input": "2024-11-05T04:49:38.070779Z", + "iopub.status.busy": "2024-11-05T04:49:38.070561Z", + "iopub.status.idle": "2024-11-05T04:49:38.079941Z", + "shell.execute_reply": "2024-11-05T04:49:38.079637Z" } }, "outputs": [ @@ -797,9 +797,9 @@ "data": { "text/html": [ "\n", - "

class MetaflowData [source]

Container of data artifacts produced by a `Task`. This object is
instantiated through `Task.data`.

`MetaflowData` allows results to be retrieved by their name
through a convenient dot notation:

```python
Task(...).data.my_object
```

You can also test the existence of an object

```python
if 'my_object' in Task(...).data:
    print('my_object found')
```

Note that this container relies on the local cache to load all data
artifacts. If your `Task` contains a lot of data, a more efficient
approach is to load artifacts individually like so

```
Task(...)['my_object'].data
```

\n", + "

class MetaflowData [source]

Container of data artifacts produced by a `Task`. This object is
instantiated through `Task.data`.

`MetaflowData` allows results to be retrieved by their name
through a convenient dot notation:

```python
Task(...).data.my_object
```

You can also test the existence of an object

```python
if 'my_object' in Task(...).data:
    print('my_object found')
```

Note that this container relies on the local cache to load all data
artifacts. If your `Task` contains a lot of data, a more efficient
approach is to load artifacts individually like so

```
Task(...)['my_object'].data
```

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -808,7 +808,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 16, @@ -837,10 +837,10 @@ "id": "5f454538", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.770085Z", - "iopub.status.busy": "2024-07-25T06:16:57.769991Z", - "iopub.status.idle": "2024-07-25T06:16:57.777954Z", - "shell.execute_reply": "2024-07-25T06:16:57.777737Z" + "iopub.execute_input": "2024-11-05T04:49:38.083198Z", + "iopub.status.busy": "2024-11-05T04:49:38.083070Z", + "iopub.status.idle": "2024-11-05T04:49:38.092678Z", + "shell.execute_reply": "2024-11-05T04:49:38.092297Z" } }, "outputs": [ @@ -848,9 +848,9 @@ "data": { "text/html": [ "\n", - "

class MetaflowCode [source]

Snapshot of the code used to execute this `Run`. Instantiate the object through
`Run(...).code` (if any step is executed remotely) or `Task(...).code` for an
individual task. The code package is the same for all steps of a `Run`.

`MetaflowCode` includes a package of the user-defined `FlowSpec` class and supporting
files, as well as a snapshot of the Metaflow library itself.

Currently, `MetaflowCode` objects are stored only for `Run`s that have at least one `Step`
executing outside the user's local environment.

The `TarFile` for the `Run` is given by `Run(...).code.tarball`

Attributes
----------
path : str
    Location (in the datastore provider) of the code package.
info : Dict[str, str]
    Dictionary of information related to this code-package.
flowspec : str
    Source code of the file containing the `FlowSpec` in this code package.
tarball : TarFile
    Python standard library `tarfile.TarFile` archive containing all the code.

\n", + "

class MetaflowCode [source]

Snapshot of the code used to execute this `Run`. Instantiate the object through
`Run(...).code` (if any step is executed remotely) or `Task(...).code` for an
individual task. The code package is the same for all steps of a `Run`.

`MetaflowCode` includes a package of the user-defined `FlowSpec` class and supporting
files, as well as a snapshot of the Metaflow library itself.

Currently, `MetaflowCode` objects are stored only for `Run`s that have at least one `Step`
executing outside the user's local environment.

The `TarFile` for the `Run` is given by `Run(...).code.tarball`

Attributes
----------
path : str
    Location (in the datastore provider) of the code package.
info : Dict[str, str]
    Dictionary of information related to this code-package.
flowspec : str
    Source code of the file containing the `FlowSpec` in this code package.
tarball : TarFile
    Python standard library `tarfile.TarFile` archive containing all the code.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -864,7 +864,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 17, @@ -906,10 +906,10 @@ "id": "b3817f2e", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.780159Z", - "iopub.status.busy": "2024-07-25T06:16:57.780077Z", - "iopub.status.idle": "2024-07-25T06:16:57.782016Z", - "shell.execute_reply": "2024-07-25T06:16:57.781773Z" + "iopub.execute_input": "2024-11-05T04:49:38.095571Z", + "iopub.status.busy": "2024-11-05T04:49:38.095419Z", + "iopub.status.idle": "2024-11-05T04:49:38.097991Z", + "shell.execute_reply": "2024-11-05T04:49:38.097597Z" } }, "outputs": [ @@ -930,7 +930,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 18, @@ -948,10 +948,10 @@ "id": "9192a187", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.784284Z", - "iopub.status.busy": "2024-07-25T06:16:57.784201Z", - "iopub.status.idle": "2024-07-25T06:16:57.785987Z", - "shell.execute_reply": "2024-07-25T06:16:57.785765Z" + "iopub.execute_input": "2024-11-05T04:49:38.101844Z", + "iopub.status.busy": "2024-11-05T04:49:38.101674Z", + "iopub.status.idle": "2024-11-05T04:49:38.104230Z", + "shell.execute_reply": "2024-11-05T04:49:38.103958Z" } }, "outputs": [ @@ -972,7 +972,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 19, @@ -990,10 +990,10 @@ "id": "97c20808", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.788415Z", - "iopub.status.busy": "2024-07-25T06:16:57.788332Z", - "iopub.status.idle": "2024-07-25T06:16:57.790307Z", - "shell.execute_reply": "2024-07-25T06:16:57.790084Z" + "iopub.execute_input": "2024-11-05T04:49:38.107542Z", + "iopub.status.busy": "2024-11-05T04:49:38.107402Z", + "iopub.status.idle": "2024-11-05T04:49:38.109879Z", + "shell.execute_reply": "2024-11-05T04:49:38.109591Z" } }, "outputs": [ @@ -1014,7 +1014,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 20, @@ -1032,10 +1032,10 @@ "id": "ba6912f9", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.792604Z", - "iopub.status.busy": "2024-07-25T06:16:57.792522Z", - "iopub.status.idle": "2024-07-25T06:16:57.794500Z", - "shell.execute_reply": "2024-07-25T06:16:57.794291Z" + "iopub.execute_input": "2024-11-05T04:49:38.114024Z", + "iopub.status.busy": "2024-11-05T04:49:38.113812Z", + "iopub.status.idle": "2024-11-05T04:49:38.117074Z", + "shell.execute_reply": "2024-11-05T04:49:38.116734Z" } }, "outputs": [ @@ -1056,7 +1056,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 21, @@ -1074,10 +1074,10 @@ "id": "dc2fb830", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.796609Z", - "iopub.status.busy": "2024-07-25T06:16:57.796507Z", - "iopub.status.idle": "2024-07-25T06:16:57.799713Z", - "shell.execute_reply": "2024-07-25T06:16:57.799471Z" + "iopub.execute_input": "2024-11-05T04:49:38.120639Z", + "iopub.status.busy": "2024-11-05T04:49:38.120481Z", + "iopub.status.idle": "2024-11-05T04:49:38.124227Z", + "shell.execute_reply": "2024-11-05T04:49:38.123718Z" } }, "outputs": [ @@ -1098,7 +1098,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 22, @@ -1126,10 +1126,10 @@ "id": "249a8772", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.801634Z", - "iopub.status.busy": "2024-07-25T06:16:57.801535Z", - "iopub.status.idle": "2024-07-25T06:16:57.804565Z", - "shell.execute_reply": "2024-07-25T06:16:57.804334Z" + "iopub.execute_input": "2024-11-05T04:49:38.127305Z", + "iopub.status.busy": "2024-11-05T04:49:38.127176Z", + "iopub.status.idle": "2024-11-05T04:49:38.130891Z", + "shell.execute_reply": "2024-11-05T04:49:38.130595Z" } }, "outputs": [ @@ -1153,7 +1153,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 23, @@ -1179,10 +1179,10 @@ "id": "a8ff9d8a", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.806732Z", - "iopub.status.busy": "2024-07-25T06:16:57.806634Z", - "iopub.status.idle": "2024-07-25T06:16:57.809064Z", - "shell.execute_reply": "2024-07-25T06:16:57.808806Z" + "iopub.execute_input": "2024-11-05T04:49:38.133846Z", + "iopub.status.busy": "2024-11-05T04:49:38.133675Z", + "iopub.status.idle": "2024-11-05T04:49:38.136905Z", + "shell.execute_reply": "2024-11-05T04:49:38.136495Z" } }, "outputs": [ @@ -1190,9 +1190,9 @@ "data": { "text/html": [ "\n", - "

function namespace (ns: Optional[str]) -> Optional[str][source]

metaflow

Switch namespace to the one provided.

This call has a global effect. No objects outside this namespace
will be accessible. To access all objects regardless of namespaces,
pass None to this call.

Parameters
----------
ns : str, optional
    Namespace to switch to or None to ignore namespaces.

Returns
-------
str, optional
    Namespace set (result of get_namespace()).

\n", + "

function namespace (ns: Optional[str]) -> Optional[str][source]

metaflow

Switch namespace to the one provided.

This call has a global effect. No objects outside this namespace
will be accessible. To access all objects regardless of namespaces,
pass None to this call.

Parameters
----------
ns : str, optional
    Namespace to switch to or None to ignore namespaces.

Returns
-------
str, optional
    Namespace set (result of get_namespace()).

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -1206,7 +1206,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 24, @@ -1226,10 +1226,10 @@ "id": "864e244f", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.811079Z", - "iopub.status.busy": "2024-07-25T06:16:57.810986Z", - "iopub.status.idle": "2024-07-25T06:16:57.813253Z", - "shell.execute_reply": "2024-07-25T06:16:57.813026Z" + "iopub.execute_input": "2024-11-05T04:49:38.140286Z", + "iopub.status.busy": "2024-11-05T04:49:38.140061Z", + "iopub.status.idle": "2024-11-05T04:49:38.143187Z", + "shell.execute_reply": "2024-11-05T04:49:38.142872Z" } }, "outputs": [ @@ -1237,9 +1237,9 @@ "data": { "text/html": [ "\n", - "

function get_namespace () -> Optional[str][source]

Return the current namespace that is currently being used to filter objects.

The namespace is a tag associated with all objects in Metaflow.

Returns
-------
str, optional
    The current namespace used to filter objects.

\n", + "

function get_namespace () -> Optional[str][source]

Return the current namespace that is currently being used to filter objects.

The namespace is a tag associated with all objects in Metaflow.

Returns
-------
str, optional
    The current namespace used to filter objects.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -1250,7 +1250,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 25, @@ -1268,10 +1268,10 @@ "id": "af4a31de", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.815463Z", - "iopub.status.busy": "2024-07-25T06:16:57.815378Z", - "iopub.status.idle": "2024-07-25T06:16:57.817523Z", - "shell.execute_reply": "2024-07-25T06:16:57.817314Z" + "iopub.execute_input": "2024-11-05T04:49:38.145635Z", + "iopub.status.busy": "2024-11-05T04:49:38.145516Z", + "iopub.status.idle": "2024-11-05T04:49:38.148039Z", + "shell.execute_reply": "2024-11-05T04:49:38.147788Z" } }, "outputs": [ @@ -1279,9 +1279,9 @@ "data": { "text/html": [ "\n", - "

function default_namespace () -> str[source]

Resets the namespace used to filter objects to the default one, i.e. the one that was
used prior to any `namespace` calls.

Returns
-------
str
    The result of get_namespace() after the namespace has been reset.

\n", + "

function default_namespace () -> str[source]

Resets the namespace used to filter objects to the default one, i.e. the one that was
used prior to any `namespace` calls.

Returns
-------
str
    The result of get_namespace() after the namespace has been reset.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -1292,7 +1292,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 26, @@ -1318,10 +1318,10 @@ "id": "4b040aaf", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.819632Z", - "iopub.status.busy": "2024-07-25T06:16:57.819555Z", - "iopub.status.idle": "2024-07-25T06:16:57.822062Z", - "shell.execute_reply": "2024-07-25T06:16:57.821854Z" + "iopub.execute_input": "2024-11-05T04:49:38.151038Z", + "iopub.status.busy": "2024-11-05T04:49:38.150907Z", + "iopub.status.idle": "2024-11-05T04:49:38.154015Z", + "shell.execute_reply": "2024-11-05T04:49:38.153550Z" } }, "outputs": [ @@ -1329,9 +1329,9 @@ "data": { "text/html": [ "\n", - "

function metadata (ms: str) -> str[source]

Switch Metadata provider.

This call has a global effect. Selecting the local metadata will,
for example, not allow access to information stored in remote
metadata providers.

Note that you don't typically have to call this function directly. Usually
the metadata provider is set through the Metaflow configuration file. If you
need to switch between multiple providers, you can use the `METAFLOW_PROFILE`
environment variable to switch between configurations.

Parameters
----------
ms : str
    Can be a path (selects local metadata), a URL starting with http (selects
    the service metadata) or an explicit specification @; as an
    example, you can specify local@ or service@.

Returns
-------
str
    The description of the metadata selected (equivalent to the result of
    get_metadata()).

\n", + "

function metadata (ms: str) -> str[source]

Switch Metadata provider.

This call has a global effect. Selecting the local metadata will,
for example, not allow access to information stored in remote
metadata providers.

Note that you don't typically have to call this function directly. Usually
the metadata provider is set through the Metaflow configuration file. If you
need to switch between multiple providers, you can use the `METAFLOW_PROFILE`
environment variable to switch between configurations.

Parameters
----------
ms : str
    Can be a path (selects local metadata), a URL starting with http (selects
    the service metadata) or an explicit specification @; as an
    example, you can specify local@ or service@.

Returns
-------
str
    The description of the metadata selected (equivalent to the result of
    get_metadata()).

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -1345,7 +1345,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 27, @@ -1365,10 +1365,10 @@ "id": "940db9bf", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.824057Z", - "iopub.status.busy": "2024-07-25T06:16:57.823981Z", - "iopub.status.idle": "2024-07-25T06:16:57.826138Z", - "shell.execute_reply": "2024-07-25T06:16:57.825906Z" + "iopub.execute_input": "2024-11-05T04:49:38.156854Z", + "iopub.status.busy": "2024-11-05T04:49:38.156728Z", + "iopub.status.idle": "2024-11-05T04:49:38.159412Z", + "shell.execute_reply": "2024-11-05T04:49:38.159109Z" } }, "outputs": [ @@ -1376,9 +1376,9 @@ "data": { "text/html": [ "\n", - "

function get_metadata () -> str[source]

Returns the current Metadata provider.

If this is not set explicitly using `metadata`, the default value is
determined through the Metaflow configuration. You can use this call to
check that your configuration is set up properly.

If multiple configuration profiles are present, this call returns the one
selected through the `METAFLOW_PROFILE` environment variable.

Returns
-------
str
    Information about the Metadata provider currently selected. This information typically
    returns provider specific information (like URL for remote providers or local paths for
    local providers).

\n", + "

function get_metadata () -> str[source]

Returns the current Metadata provider.

If this is not set explicitly using `metadata`, the default value is
determined through the Metaflow configuration. You can use this call to
check that your configuration is set up properly.

If multiple configuration profiles are present, this call returns the one
selected through the `METAFLOW_PROFILE` environment variable.

Returns
-------
str
    Information about the Metadata provider currently selected. This information typically
    returns provider specific information (like URL for remote providers or local paths for
    local providers).

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -1389,7 +1389,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 28, @@ -1407,10 +1407,10 @@ "id": "4fa4b981", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.828277Z", - "iopub.status.busy": "2024-07-25T06:16:57.828192Z", - "iopub.status.idle": "2024-07-25T06:16:57.830401Z", - "shell.execute_reply": "2024-07-25T06:16:57.830156Z" + "iopub.execute_input": "2024-11-05T04:49:38.162177Z", + "iopub.status.busy": "2024-11-05T04:49:38.162057Z", + "iopub.status.idle": "2024-11-05T04:49:38.164618Z", + "shell.execute_reply": "2024-11-05T04:49:38.164325Z" } }, "outputs": [ @@ -1418,9 +1418,9 @@ "data": { "text/html": [ "\n", - "

function default_metadata () -> str[source]

Resets the Metadata provider to the default value, that is, to the value
that was used prior to any `metadata` calls.

Returns
-------
str
    The result of get_metadata() after resetting the provider.

\n", + "

function default_metadata () -> str[source]

Resets the Metadata provider to the default value, that is, to the value
that was used prior to any `metadata` calls.

Returns
-------
str
    The result of get_metadata() after resetting the provider.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -1431,7 +1431,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 29, @@ -1468,7 +1468,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0" + "version": "3.10.13" } }, "nbformat": 4, diff --git a/docs/api/client.md b/docs/api/client.md index ccbc679e..045e1403 100644 --- a/docs/api/client.md +++ b/docs/api/client.md @@ -57,9 +57,9 @@ This module accesses all objects through the current metadata provider - either ### Metaflow - + - + @@ -71,7 +71,7 @@ This module accesses all objects through the current metadata provider - either ### Flow - + @@ -84,7 +84,7 @@ This module accesses all objects through the current metadata provider - either - + @@ -101,7 +101,7 @@ This module accesses all objects through the current metadata provider - either ### Run - + @@ -119,7 +119,7 @@ This module accesses all objects through the current metadata provider - either - + @@ -131,7 +131,7 @@ This module accesses all objects through the current metadata provider - either - + @@ -143,7 +143,7 @@ This module accesses all objects through the current metadata provider - either - + @@ -155,7 +155,7 @@ This module accesses all objects through the current metadata provider - either - + @@ -167,7 +167,7 @@ This module accesses all objects through the current metadata provider - either - + @@ -180,7 +180,7 @@ This module accesses all objects through the current metadata provider - either - + @@ -195,7 +195,7 @@ This module accesses all objects through the current metadata provider - either ### Step - + @@ -211,7 +211,7 @@ This module accesses all objects through the current metadata provider - either ### Task - + @@ -235,7 +235,7 @@ This module accesses all objects through the current metadata provider - either - + @@ -253,7 +253,7 @@ This module accesses all objects through the current metadata provider - either ### DataArtifact - + @@ -271,7 +271,7 @@ This module accesses all objects through the current metadata provider - either ### MetaflowData - + @@ -283,7 +283,7 @@ This module accesses all objects through the current metadata provider - either ### MetaflowCode - + @@ -383,7 +383,7 @@ This module accesses all objects through the current metadata provider - either ## Namespace functions - + @@ -398,7 +398,7 @@ This module accesses all objects through the current metadata provider - either - + @@ -410,7 +410,7 @@ This module accesses all objects through the current metadata provider - either - + @@ -424,7 +424,7 @@ This module accesses all objects through the current metadata provider - either ## Metadata functions - + @@ -439,7 +439,7 @@ This module accesses all objects through the current metadata provider - either - + @@ -451,7 +451,7 @@ This module accesses all objects through the current metadata provider - either - + diff --git a/docs/api/current.ipynb b/docs/api/current.ipynb index a038653d..d6c11fa3 100644 --- a/docs/api/current.ipynb +++ b/docs/api/current.ipynb @@ -24,10 +24,10 @@ "id": "d4c02781", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:55.863821Z", - "iopub.status.busy": "2024-07-25T06:16:55.863687Z", - "iopub.status.idle": "2024-07-25T06:16:56.135122Z", - "shell.execute_reply": "2024-07-25T06:16:56.134781Z" + "iopub.execute_input": "2024-11-05T04:49:37.241508Z", + "iopub.status.busy": "2024-11-05T04:49:37.240993Z", + "iopub.status.idle": "2024-11-05T04:49:37.598461Z", + "shell.execute_reply": "2024-11-05T04:49:37.597912Z" } }, "outputs": [], @@ -182,10 +182,10 @@ "id": "7fdb7184", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.137589Z", - "iopub.status.busy": "2024-07-25T06:16:56.137412Z", - "iopub.status.idle": "2024-07-25T06:16:56.141437Z", - "shell.execute_reply": "2024-07-25T06:16:56.141114Z" + "iopub.execute_input": "2024-11-05T04:49:37.602303Z", + "iopub.status.busy": "2024-11-05T04:49:37.602034Z", + "iopub.status.idle": "2024-11-05T04:49:37.607505Z", + "shell.execute_reply": "2024-11-05T04:49:37.607024Z" } }, "outputs": [ @@ -204,7 +204,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -222,10 +222,10 @@ "id": "fa61bd9b", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.143388Z", - "iopub.status.busy": "2024-07-25T06:16:56.143306Z", - "iopub.status.idle": "2024-07-25T06:16:56.145573Z", - "shell.execute_reply": "2024-07-25T06:16:56.145253Z" + "iopub.execute_input": "2024-11-05T04:49:37.611134Z", + "iopub.status.busy": "2024-11-05T04:49:37.610954Z", + "iopub.status.idle": "2024-11-05T04:49:37.613868Z", + "shell.execute_reply": "2024-11-05T04:49:37.613461Z" } }, "outputs": [ @@ -244,7 +244,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 3, @@ -262,10 +262,10 @@ "id": "16a90815", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.147932Z", - "iopub.status.busy": "2024-07-25T06:16:56.147823Z", - "iopub.status.idle": "2024-07-25T06:16:56.150134Z", - "shell.execute_reply": "2024-07-25T06:16:56.149852Z" + "iopub.execute_input": "2024-11-05T04:49:37.618577Z", + "iopub.status.busy": "2024-11-05T04:49:37.618355Z", + "iopub.status.idle": "2024-11-05T04:49:37.622018Z", + "shell.execute_reply": "2024-11-05T04:49:37.621558Z" } }, "outputs": [ @@ -284,7 +284,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 4, @@ -302,10 +302,10 @@ "id": "f278667f", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.152471Z", - "iopub.status.busy": "2024-07-25T06:16:56.152367Z", - "iopub.status.idle": "2024-07-25T06:16:56.154533Z", - "shell.execute_reply": "2024-07-25T06:16:56.154202Z" + "iopub.execute_input": "2024-11-05T04:49:37.626276Z", + "iopub.status.busy": "2024-11-05T04:49:37.626091Z", + "iopub.status.idle": "2024-11-05T04:49:37.629329Z", + "shell.execute_reply": "2024-11-05T04:49:37.628714Z" } }, "outputs": [ @@ -324,7 +324,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 5, @@ -342,10 +342,10 @@ "id": "f4be98ad", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.157036Z", - "iopub.status.busy": "2024-07-25T06:16:56.156950Z", - "iopub.status.idle": "2024-07-25T06:16:56.158869Z", - "shell.execute_reply": "2024-07-25T06:16:56.158635Z" + "iopub.execute_input": "2024-11-05T04:49:37.633427Z", + "iopub.status.busy": "2024-11-05T04:49:37.633293Z", + "iopub.status.idle": "2024-11-05T04:49:37.637251Z", + "shell.execute_reply": "2024-11-05T04:49:37.636301Z" } }, "outputs": [ @@ -364,7 +364,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 6, @@ -382,10 +382,10 @@ "id": "a97ad305", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.161044Z", - "iopub.status.busy": "2024-07-25T06:16:56.160962Z", - "iopub.status.idle": "2024-07-25T06:16:56.163040Z", - "shell.execute_reply": "2024-07-25T06:16:56.162796Z" + "iopub.execute_input": "2024-11-05T04:49:37.640944Z", + "iopub.status.busy": "2024-11-05T04:49:37.640775Z", + "iopub.status.idle": "2024-11-05T04:49:37.644501Z", + "shell.execute_reply": "2024-11-05T04:49:37.643575Z" } }, "outputs": [ @@ -404,7 +404,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 7, @@ -422,10 +422,10 @@ "id": "b73e163c", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.165295Z", - "iopub.status.busy": "2024-07-25T06:16:56.165198Z", - "iopub.status.idle": "2024-07-25T06:16:56.167437Z", - "shell.execute_reply": "2024-07-25T06:16:56.167116Z" + "iopub.execute_input": "2024-11-05T04:49:37.649647Z", + "iopub.status.busy": "2024-11-05T04:49:37.649384Z", + "iopub.status.idle": "2024-11-05T04:49:37.652994Z", + "shell.execute_reply": "2024-11-05T04:49:37.652555Z" } }, "outputs": [ @@ -444,7 +444,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 8, @@ -462,10 +462,10 @@ "id": "16dd0a74", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.169732Z", - "iopub.status.busy": "2024-07-25T06:16:56.169604Z", - "iopub.status.idle": "2024-07-25T06:16:56.171888Z", - "shell.execute_reply": "2024-07-25T06:16:56.171650Z" + "iopub.execute_input": "2024-11-05T04:49:37.657598Z", + "iopub.status.busy": "2024-11-05T04:49:37.657435Z", + "iopub.status.idle": "2024-11-05T04:49:37.660209Z", + "shell.execute_reply": "2024-11-05T04:49:37.659849Z" } }, "outputs": [ @@ -484,7 +484,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 9, @@ -502,10 +502,10 @@ "id": "8f758f03", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.174219Z", - "iopub.status.busy": "2024-07-25T06:16:56.174131Z", - "iopub.status.idle": "2024-07-25T06:16:56.176053Z", - "shell.execute_reply": "2024-07-25T06:16:56.175835Z" + "iopub.execute_input": "2024-11-05T04:49:37.664192Z", + "iopub.status.busy": "2024-11-05T04:49:37.664042Z", + "iopub.status.idle": "2024-11-05T04:49:37.667594Z", + "shell.execute_reply": "2024-11-05T04:49:37.667139Z" } }, "outputs": [ @@ -524,7 +524,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 10, @@ -542,10 +542,10 @@ "id": "feaa6830", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.178305Z", - "iopub.status.busy": "2024-07-25T06:16:56.178226Z", - "iopub.status.idle": "2024-07-25T06:16:56.180109Z", - "shell.execute_reply": "2024-07-25T06:16:56.179858Z" + "iopub.execute_input": "2024-11-05T04:49:37.672168Z", + "iopub.status.busy": "2024-11-05T04:49:37.671946Z", + "iopub.status.idle": "2024-11-05T04:49:37.674865Z", + "shell.execute_reply": "2024-11-05T04:49:37.674513Z" } }, "outputs": [ @@ -564,7 +564,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 11, @@ -582,10 +582,10 @@ "id": "66c47292", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.182576Z", - "iopub.status.busy": "2024-07-25T06:16:56.182485Z", - "iopub.status.idle": "2024-07-25T06:16:56.184502Z", - "shell.execute_reply": "2024-07-25T06:16:56.184224Z" + "iopub.execute_input": "2024-11-05T04:49:37.679813Z", + "iopub.status.busy": "2024-11-05T04:49:37.679626Z", + "iopub.status.idle": "2024-11-05T04:49:37.682344Z", + "shell.execute_reply": "2024-11-05T04:49:37.682002Z" } }, "outputs": [ @@ -604,7 +604,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 12, @@ -642,10 +642,10 @@ "id": "437b253d", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.187401Z", - "iopub.status.busy": "2024-07-25T06:16:56.187297Z", - "iopub.status.idle": "2024-07-25T06:16:56.189596Z", - "shell.execute_reply": "2024-07-25T06:16:56.189229Z" + "iopub.execute_input": "2024-11-05T04:49:37.686280Z", + "iopub.status.busy": "2024-11-05T04:49:37.686154Z", + "iopub.status.idle": "2024-11-05T04:49:37.706723Z", + "shell.execute_reply": "2024-11-05T04:49:37.701108Z" } }, "outputs": [ @@ -664,7 +664,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 13, @@ -682,10 +682,10 @@ "id": "fe5e9e0a", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.194090Z", - "iopub.status.busy": "2024-07-25T06:16:56.193991Z", - "iopub.status.idle": "2024-07-25T06:16:56.196057Z", - "shell.execute_reply": "2024-07-25T06:16:56.195807Z" + "iopub.execute_input": "2024-11-05T04:49:37.724293Z", + "iopub.status.busy": "2024-11-05T04:49:37.723835Z", + "iopub.status.idle": "2024-11-05T04:49:37.728562Z", + "shell.execute_reply": "2024-11-05T04:49:37.728088Z" } }, "outputs": [ @@ -704,7 +704,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 14, @@ -722,10 +722,10 @@ "id": "1097fb92", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.198372Z", - "iopub.status.busy": "2024-07-25T06:16:56.198283Z", - "iopub.status.idle": "2024-07-25T06:16:56.200219Z", - "shell.execute_reply": "2024-07-25T06:16:56.199992Z" + "iopub.execute_input": "2024-11-05T04:49:37.738476Z", + "iopub.status.busy": "2024-11-05T04:49:37.738273Z", + "iopub.status.idle": "2024-11-05T04:49:37.740960Z", + "shell.execute_reply": "2024-11-05T04:49:37.740616Z" } }, "outputs": [ @@ -744,7 +744,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 15, @@ -762,10 +762,10 @@ "id": "836d028b", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.202408Z", - "iopub.status.busy": "2024-07-25T06:16:56.202327Z", - "iopub.status.idle": "2024-07-25T06:16:56.204279Z", - "shell.execute_reply": "2024-07-25T06:16:56.204025Z" + "iopub.execute_input": "2024-11-05T04:49:37.745069Z", + "iopub.status.busy": "2024-11-05T04:49:37.744826Z", + "iopub.status.idle": "2024-11-05T04:49:37.747396Z", + "shell.execute_reply": "2024-11-05T04:49:37.747044Z" } }, "outputs": [ @@ -784,7 +784,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 16, @@ -802,10 +802,10 @@ "id": "96348e52", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.206534Z", - "iopub.status.busy": "2024-07-25T06:16:56.206443Z", - "iopub.status.idle": "2024-07-25T06:16:56.208449Z", - "shell.execute_reply": "2024-07-25T06:16:56.208178Z" + "iopub.execute_input": "2024-11-05T04:49:37.752253Z", + "iopub.status.busy": "2024-11-05T04:49:37.752120Z", + "iopub.status.idle": "2024-11-05T04:49:37.754653Z", + "shell.execute_reply": "2024-11-05T04:49:37.754316Z" } }, "outputs": [ @@ -824,7 +824,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 17, @@ -853,10 +853,10 @@ "id": "640602e7", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.210527Z", - "iopub.status.busy": "2024-07-25T06:16:56.210441Z", - "iopub.status.idle": "2024-07-25T06:16:56.212903Z", - "shell.execute_reply": "2024-07-25T06:16:56.212663Z" + "iopub.execute_input": "2024-11-05T04:49:37.758434Z", + "iopub.status.busy": "2024-11-05T04:49:37.758293Z", + "iopub.status.idle": "2024-11-05T04:49:37.761524Z", + "shell.execute_reply": "2024-11-05T04:49:37.761153Z" } }, "outputs": [ @@ -880,7 +880,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 18, @@ -898,10 +898,10 @@ "id": "0709cf75", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.214782Z", - "iopub.status.busy": "2024-07-25T06:16:56.214699Z", - "iopub.status.idle": "2024-07-25T06:16:56.216878Z", - "shell.execute_reply": "2024-07-25T06:16:56.216624Z" + "iopub.execute_input": "2024-11-05T04:49:37.765064Z", + "iopub.status.busy": "2024-11-05T04:49:37.764923Z", + "iopub.status.idle": "2024-11-05T04:49:37.767709Z", + "shell.execute_reply": "2024-11-05T04:49:37.767424Z" } }, "outputs": [ @@ -923,7 +923,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 19, @@ -941,10 +941,10 @@ "id": "7ef69ab6", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.218955Z", - "iopub.status.busy": "2024-07-25T06:16:56.218859Z", - "iopub.status.idle": "2024-07-25T06:16:56.220971Z", - "shell.execute_reply": "2024-07-25T06:16:56.220735Z" + "iopub.execute_input": "2024-11-05T04:49:37.770917Z", + "iopub.status.busy": "2024-11-05T04:49:37.770736Z", + "iopub.status.idle": "2024-11-05T04:49:37.773703Z", + "shell.execute_reply": "2024-11-05T04:49:37.773309Z" } }, "outputs": [ @@ -965,7 +965,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 20, @@ -983,10 +983,10 @@ "id": "f667ebcd", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.223189Z", - "iopub.status.busy": "2024-07-25T06:16:56.223109Z", - "iopub.status.idle": "2024-07-25T06:16:56.225221Z", - "shell.execute_reply": "2024-07-25T06:16:56.224988Z" + "iopub.execute_input": "2024-11-05T04:49:37.777569Z", + "iopub.status.busy": "2024-11-05T04:49:37.777432Z", + "iopub.status.idle": "2024-11-05T04:49:37.780329Z", + "shell.execute_reply": "2024-11-05T04:49:37.779991Z" } }, "outputs": [ @@ -1007,7 +1007,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 21, @@ -1025,10 +1025,10 @@ "id": "6d27efa0", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.227442Z", - "iopub.status.busy": "2024-07-25T06:16:56.227348Z", - "iopub.status.idle": "2024-07-25T06:16:56.229481Z", - "shell.execute_reply": "2024-07-25T06:16:56.229231Z" + "iopub.execute_input": "2024-11-05T04:49:37.783562Z", + "iopub.status.busy": "2024-11-05T04:49:37.783401Z", + "iopub.status.idle": "2024-11-05T04:49:37.786031Z", + "shell.execute_reply": "2024-11-05T04:49:37.785748Z" } }, "outputs": [ @@ -1047,7 +1047,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 22, @@ -1065,10 +1065,10 @@ "id": "a7914220", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.231731Z", - "iopub.status.busy": "2024-07-25T06:16:56.231634Z", - "iopub.status.idle": "2024-07-25T06:16:56.233757Z", - "shell.execute_reply": "2024-07-25T06:16:56.233513Z" + "iopub.execute_input": "2024-11-05T04:49:37.790200Z", + "iopub.status.busy": "2024-11-05T04:49:37.790036Z", + "iopub.status.idle": "2024-11-05T04:49:37.792746Z", + "shell.execute_reply": "2024-11-05T04:49:37.792425Z" } }, "outputs": [ @@ -1087,7 +1087,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 23, @@ -1116,10 +1116,10 @@ "id": "2ad67aa6", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.235976Z", - "iopub.status.busy": "2024-07-25T06:16:56.235885Z", - "iopub.status.idle": "2024-07-25T06:16:56.237887Z", - "shell.execute_reply": "2024-07-25T06:16:56.237650Z" + "iopub.execute_input": "2024-11-05T04:49:37.796749Z", + "iopub.status.busy": "2024-11-05T04:49:37.796598Z", + "iopub.status.idle": "2024-11-05T04:49:37.799094Z", + "shell.execute_reply": "2024-11-05T04:49:37.798747Z" } }, "outputs": [ @@ -1138,7 +1138,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 24, @@ -1175,7 +1175,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0" + "version": "3.10.13" } }, "nbformat": 4, diff --git a/docs/api/deployer.ipynb b/docs/api/deployer.ipynb index 497b00bc..5723ba1f 100644 --- a/docs/api/deployer.ipynb +++ b/docs/api/deployer.ipynb @@ -22,10 +22,10 @@ "id": "d4c02781", "metadata": { "execution": { - "iopub.execute_input": "2024-05-28T03:59:53.292277Z", - "iopub.status.busy": "2024-05-28T03:59:53.292089Z", - "iopub.status.idle": "2024-05-28T03:59:53.599154Z", - "shell.execute_reply": "2024-05-28T03:59:53.598652Z" + "iopub.execute_input": "2024-11-05T04:49:36.083072Z", + "iopub.status.busy": "2024-11-05T04:49:36.082332Z", + "iopub.status.idle": "2024-11-05T04:49:36.442175Z", + "shell.execute_reply": "2024-11-05T04:49:36.441203Z" } }, "outputs": [], @@ -36,7 +36,7 @@ "ShowDoc = partial(ShowDoc, module_nm='metaflow')\n", "\n", "from metaflow import Deployer as RealDeployer\n", - "from metaflow.runner.deployer import DeployerImpl\n", + "from metaflow.runner.deployer_impl import DeployerImpl\n", "from metaflow.runner.deployer import TriggeredRun\n", "\n", "class Deployer:\n", @@ -114,10 +114,10 @@ "id": "7fdb7184", "metadata": { "execution": { - "iopub.execute_input": "2024-05-28T03:59:53.602322Z", - "iopub.status.busy": "2024-05-28T03:59:53.602100Z", - "iopub.status.idle": "2024-05-28T03:59:53.610438Z", - "shell.execute_reply": "2024-05-28T03:59:53.610130Z" + "iopub.execute_input": "2024-11-05T04:49:36.447563Z", + "iopub.status.busy": "2024-11-05T04:49:36.447294Z", + "iopub.status.idle": "2024-11-05T04:49:36.458600Z", + "shell.execute_reply": "2024-11-05T04:49:36.457522Z" } }, "outputs": [ @@ -125,7 +125,7 @@ "data": { "text/html": [ "\n", - "

class Deployer flow_file, show_output=True, profile=None, env=None, cwd=None, **kwargs[source]

Use the `Deployer` class to configure and access one of the production
orchestrators supported by Metaflow.

Parameters
----------
flow_file : str
    Path to the flow file to deploy.
show_output : bool, default True
    Show the 'stdout' and 'stderr' to the console by default.
profile : Optional[str], default None
    Metaflow profile to use for the deployment. If not specified, the default
    profile is used.
env : Optional[Dict[str, str]], default None
    Additional environment variables to set for the deployment.
cwd : Optional[str], default None
    The directory to run the subprocess in; if not specified, the current
    directory is used.
**kwargs : Any
    Additional arguments that you would pass to `python myflow.py` before
    the deployment command.

\n", + "

class Deployer flow_file, show_output=True, profile=None, env=None, cwd=None, **kwargs[source]

Use the `Deployer` class to configure and access one of the production
orchestrators supported by Metaflow.

Parameters
----------
flow_file : str
    Path to the flow file to deploy.
show_output : bool, default True
    Show the 'stdout' and 'stderr' to the console by default.
profile : Optional[str], default None
    Metaflow profile to use for the deployment. If not specified, the default
    profile is used.
env : Optional[Dict[str, str]], default None
    Additional environment variables to set for the deployment.
cwd : Optional[str], default None
    The directory to run the subprocess in; if not specified, the current
    directory is used.
file_read_timeout : int, default 3600
    The timeout until which we try to read the deployer attribute file.
**kwargs : Any
    Additional arguments that you would pass to `python myflow.py` before
    the deployment command.

\n", "\n", "\n", "\n", @@ -138,12 +138,13 @@ "\t\n", "\t\n", "\t\n", + "\t\n", "\t\n", "\n", "" ], "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -161,10 +162,10 @@ "id": "84c83820", "metadata": { "execution": { - "iopub.execute_input": "2024-05-28T03:59:53.612888Z", - "iopub.status.busy": "2024-05-28T03:59:53.612783Z", - "iopub.status.idle": "2024-05-28T03:59:53.615668Z", - "shell.execute_reply": "2024-05-28T03:59:53.615357Z" + "iopub.execute_input": "2024-11-05T04:49:36.464661Z", + "iopub.status.busy": "2024-11-05T04:49:36.464460Z", + "iopub.status.idle": "2024-11-05T04:49:36.469259Z", + "shell.execute_reply": "2024-11-05T04:49:36.468505Z" } }, "outputs": [ @@ -185,7 +186,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 3, @@ -202,6 +203,12 @@ "execution_count": 4, "id": "eb814b83", "metadata": { + "execution": { + "iopub.execute_input": "2024-11-05T04:49:36.477499Z", + "iopub.status.busy": "2024-11-05T04:49:36.477010Z", + "iopub.status.idle": "2024-11-05T04:49:36.483455Z", + "shell.execute_reply": "2024-11-05T04:49:36.482585Z" + }, "scrolled": true }, "outputs": [ @@ -222,7 +229,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 4, @@ -246,32 +253,52 @@ "cell_type": "code", "execution_count": 5, "id": "e42db208", - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2024-11-05T04:49:36.491357Z", + "iopub.status.busy": "2024-11-05T04:49:36.491147Z", + "iopub.status.idle": "2024-11-05T04:49:36.501680Z", + "shell.execute_reply": "2024-11-05T04:49:36.500672Z" + } + }, "outputs": [ { "data": { "text/html": [ "\n", - "

method ArgoWorkflowsDeployer.create (self, **kwargs)[source]

Create a deployed flow using the deployer implementation.

Parameters
----------
**kwargs : Any
    Additional arguments to pass to `create` corresponding to the
    command line arguments of `create`

Returns
-------
DeployedFlow
    DeployedFlow object representing the deployed flow.

Raises
------
Exception
    If there is an error during deployment.

\n", + "

method ArgoWorkflowsDeployer.create (self, **kwargs) -> 'metaflow.plugins.argo.argo_workflows_deployer_objects.ArgoWorkflowsDeployedFlow'[source]

Create a new ArgoWorkflow deployment.

Parameters
----------
authorize : str, optional, default None
    Authorize using this production token. Required when re-deploying an existing flow
    for the first time. The token is cached in METAFLOW_HOME.
generate_new_token : bool, optional, default False
    Generate a new production token for this flow. Moves the production flow to a new namespace.
given_token : str, optional, default None
    Use the given production token for this flow. Moves the production flow to the given namespace.
tags : List[str], optional, default None
    Annotate all objects produced by Argo Workflows runs with these tags.
user_namespace : str, optional, default None
    Change the namespace from the default (production token) to the given tag.
only_json : bool, optional, default False
    Only print out JSON sent to Argo Workflows without deploying anything.
max_workers : int, optional, default 100
    Maximum number of parallel processes.
workflow_timeout : int, optional, default None
    Workflow timeout in seconds.
workflow_priority : int, optional, default None
    Workflow priority as an integer. Higher priority workflows are processed first
    if Argo Workflows controller is configured to process limited parallel workflows.
auto_emit_argo_events : bool, optional, default True
    Auto emits Argo Events when the run completes successfully.
notify_on_error : bool, optional, default False
    Notify if the workflow fails.
notify_on_success : bool, optional, default False
    Notify if the workflow succeeds.
notify_slack_webhook_url : str, optional, default ''
    Slack incoming webhook url for workflow success/failure notifications.
notify_pager_duty_integration_key : str, optional, default ''
    PagerDuty Events API V2 Integration key for workflow success/failure notifications.
enable_heartbeat_daemon : bool, optional, default False
    Use a daemon container to broadcast heartbeats.
deployer_attribute_file : str, optional, default None
    Write the workflow name to the specified file. Used internally for Metaflow's Deployer API.
enable_error_msg_capture : bool, optional, default True
    Capture stack trace of first failed task in exit hook.

Returns
-------
ArgoWorkflowsDeployedFlow
    The Flow deployed to Argo Workflows.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", - "\n", + "\n", "\n", - "\t\n", + "\t\n", + "\t\n", + "\t\n", + "\t\n", + "\t\n", + "\t\n", + "\t\n", + "\t\n", + "\t\n", + "\t\n", + "\t\n", + "\t\n", + "\t\n", + "\t\n", + "\t\n", + "\t\n", + "\t\n", "\n", "\n", - "\t\n", - "\n", - "\n", - "\t\n", + "\t\n", "\n", "" ], "text/plain": [ - "" + "" ] }, "execution_count": 5, @@ -280,61 +307,9 @@ } ], "source": [ - "from metaflow.plugins.argo.argo_workflows_deployer import production_token as argo_production_token\n", - "from metaflow.plugins.argo.argo_workflows_deployer import delete as argo_delete\n", - "from metaflow.plugins.argo.argo_workflows_deployer import trigger as argo_trigger\n", - "from metaflow.plugins.argo.argo_workflows_deployer import terminate as argo_terminate\n", - "from metaflow.plugins.argo.argo_workflows_deployer import suspend as argo_suspend\n", - "from metaflow.plugins.argo.argo_workflows_deployer import unsuspend as argo_unsuspend\n", - "from metaflow.plugins.argo.argo_workflows_deployer import status as argo_status\n", - "\n", - "class ArgoWorkflowsTriggeredRun:\n", - " \n", - " @property\n", - " def run(self):\n", - " pass\n", - " \n", - " def terminate(self):\n", - " pass\n", - " \n", - " def suspend(self):\n", - " pass\n", - " \n", - " def unsuspend(self):\n", - " pass\n", - " \n", - " def status(self):\n", - " pass\n", - " \n", - "ArgoWorkflowsTriggeredRun.__doc__ = TriggeredRun.__doc__\n", - "ArgoWorkflowsTriggeredRun.run.__doc__ = TriggeredRun.run.__doc__\n", - "ArgoWorkflowsTriggeredRun.terminate.__doc__ = argo_terminate.__doc__\n", - "ArgoWorkflowsTriggeredRun.suspend.__doc__ = argo_suspend.__doc__\n", - "ArgoWorkflowsTriggeredRun.unsuspend.__doc__ = argo_unsuspend.__doc__\n", - "ArgoWorkflowsTriggeredRun.status.__doc__ = argo_status.__doc__\n", - "\n", - "\n", - "class ArgoWorkflowsDeployedFlow:\n", - " @property\n", - " def production_token(self):\n", - " pass\n", - " \n", - " def trigger(self) -> 'ArgoWorkflowsTriggeredRun':\n", - " pass\n", - " \n", - " def delete(self):\n", - " pass\n", - "\n", - " \n", - "ArgoWorkflowsDeployedFlow.production_token.__doc__ = argo_production_token.__doc__\n", - "ArgoWorkflowsDeployedFlow.trigger.__doc__ = argo_trigger.__doc__\n", - "ArgoWorkflowsDeployedFlow.delete.__doc__ = argo_delete.__doc__\n", - "\n", - "class ArgoWorkflowsDeployer:\n", - " def create(self, **kwargs):\n", - " pass\n", - "\n", - "ArgoWorkflowsDeployer.create.__doc__ = DeployerImpl.create.__doc__\n", + "from metaflow.plugins.argo.argo_workflows_deployer_objects import ArgoWorkflowsTriggeredRun\n", + "from metaflow.plugins.argo.argo_workflows_deployer_objects import ArgoWorkflowsDeployedFlow\n", + "from metaflow.plugins.argo.argo_workflows_deployer import ArgoWorkflowsDeployer\n", "\n", "ShowDoc(ArgoWorkflowsDeployer.create)" ] @@ -352,6 +327,12 @@ "execution_count": 6, "id": "ec96e44b", "metadata": { + "execution": { + "iopub.execute_input": "2024-11-05T04:49:36.506770Z", + "iopub.status.busy": "2024-11-05T04:49:36.506423Z", + "iopub.status.idle": "2024-11-05T04:49:36.515712Z", + "shell.execute_reply": "2024-11-05T04:49:36.513477Z" + }, "scrolled": true }, "outputs": [ @@ -359,9 +340,9 @@ "data": { "text/html": [ "\n", - "

property ArgoWorkflowsDeployedFlow.production_token

Get the production token for the deployed flow.

Returns
-------
str, optional
    The production token, None if it cannot be retrieved.

\n", + "

property ArgoWorkflowsDeployedFlow.production_token [source]

Get the production token for the deployed flow.

Returns
-------
str, optional
    The production token, None if it cannot be retrieved.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -370,7 +351,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 6, @@ -387,6 +368,12 @@ "execution_count": 7, "id": "102d67b4", "metadata": { + "execution": { + "iopub.execute_input": "2024-11-05T04:49:36.522802Z", + "iopub.status.busy": "2024-11-05T04:49:36.522522Z", + "iopub.status.idle": "2024-11-05T04:49:36.530660Z", + "shell.execute_reply": "2024-11-05T04:49:36.529630Z" + }, "scrolled": true }, "outputs": [ @@ -394,15 +381,15 @@ "data": { "text/html": [ "\n", - "

method ArgoWorkflowsDeployedFlow.trigger (self) -> 'ArgoWorkflowsTriggeredRun'[source]

Trigger a new run for the deployed flow.

Parameters
----------
**kwargs : Any
    Additional arguments to pass to the trigger command, `Parameters` in particular

Returns
-------
ArgoWorkflowsTriggeredRun
    The triggered run instance.

Raises
------
Exception
    If there is an error during the trigger process.

\n", + "

method ArgoWorkflowsDeployedFlow.trigger (self, **kwargs) -> metaflow.plugins.argo.argo_workflows_deployer_objects.ArgoWorkflowsTriggeredRun[source]

Trigger a new run for the deployed flow.

Parameters
----------
**kwargs : Any
    Additional arguments to pass to the trigger command,
    `Parameters` in particular.

Returns
-------
ArgoWorkflowsTriggeredRun
    The triggered run instance.

Raises
------
Exception
    If there is an error during the trigger process.

\n", "\n", - "\n", + "\n", "\n", - "\n", + "\n", "\n", "\n", "\n", - "\t\n", + "\t\n", "\n", "\n", "\t\n", @@ -413,7 +400,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 7, @@ -429,21 +416,28 @@ "cell_type": "code", "execution_count": 8, "id": "0cdcc724", - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2024-11-05T04:49:36.537166Z", + "iopub.status.busy": "2024-11-05T04:49:36.536927Z", + "iopub.status.idle": "2024-11-05T04:49:36.543426Z", + "shell.execute_reply": "2024-11-05T04:49:36.542638Z" + } + }, "outputs": [ { "data": { "text/html": [ "\n", - "

method ArgoWorkflowsDeployedFlow.delete (self)[source]

Delete the deployed flow.

Parameters
----------
**kwargs : Any
    Additional arguments to pass to the delete command.

Returns
-------
bool
    True if the command was successful, False otherwise.

\n", + "

method ArgoWorkflowsDeployedFlow.delete (self, **kwargs) -> bool[source]

Delete the deployed workflow template.

Parameters
----------
authorize : str, optional, default None
    Authorize the deletion with a production token.

Returns
-------
bool
    True if the command was successful, False otherwise.

\n", "\n", - "\n", + "\n", "\n", - "\n", + "\n", "\n", - "\n", + "\n", "\n", - "\t\n", + "\t\n", "\n", "\n", "\t\n", @@ -451,7 +445,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 8, @@ -476,6 +470,12 @@ "execution_count": 9, "id": "91ecd0e3", "metadata": { + "execution": { + "iopub.execute_input": "2024-11-05T04:49:36.551210Z", + "iopub.status.busy": "2024-11-05T04:49:36.550963Z", + "iopub.status.idle": "2024-11-05T04:49:36.557143Z", + "shell.execute_reply": "2024-11-05T04:49:36.555788Z" + }, "scrolled": true }, "outputs": [ @@ -483,9 +483,9 @@ "data": { "text/html": [ "\n", - "

property ArgoWorkflowsTriggeredRun.run

Retrieve the `Run` object for the triggered run.

Note that Metaflow `Run` becomes available only when the `start` task
has started executing.

Returns
-------
Run, optional
    Metaflow Run object if the `start` step has started executing, otherwise None.

\n", + "

property TriggeredRun.run [source]

Retrieve the `Run` object for the triggered run.

Note that Metaflow `Run` becomes available only when the `start` task
has started executing.

Returns
-------
Run, optional
    Metaflow Run object if the `start` step has started executing, otherwise None.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -494,7 +494,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 9, @@ -511,6 +511,12 @@ "execution_count": 10, "id": "48a0300d", "metadata": { + "execution": { + "iopub.execute_input": "2024-11-05T04:49:36.564938Z", + "iopub.status.busy": "2024-11-05T04:49:36.564478Z", + "iopub.status.idle": "2024-11-05T04:49:36.570063Z", + "shell.execute_reply": "2024-11-05T04:49:36.569184Z" + }, "scrolled": true }, "outputs": [ @@ -518,15 +524,15 @@ "data": { "text/html": [ "\n", - "

method ArgoWorkflowsTriggeredRun.terminate (self)[source]

Terminate the running workflow.

Parameters
----------
**kwargs : Any
    Additional arguments to pass to the terminate command.

Returns
-------
bool
    True if the command was successful, False otherwise.

\n", + "

method ArgoWorkflowsTriggeredRun.terminate (self, **kwargs) -> bool[source]

Terminate the running workflow.

Parameters
----------
authorize : str, optional, default None
    Authorize the termination with a production token.

Returns
-------
bool
    True if the command was successful, False otherwise.

\n", "\n", - "\n", + "\n", "\n", - "\n", + "\n", "\n", "\n", "\n", - "\t\n", + "\t\n", "\n", "\n", "\t\n", @@ -534,7 +540,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 10, @@ -551,6 +557,12 @@ "execution_count": 11, "id": "48a9708f", "metadata": { + "execution": { + "iopub.execute_input": "2024-11-05T04:49:36.575732Z", + "iopub.status.busy": "2024-11-05T04:49:36.575487Z", + "iopub.status.idle": "2024-11-05T04:49:36.580951Z", + "shell.execute_reply": "2024-11-05T04:49:36.579932Z" + }, "scrolled": true }, "outputs": [ @@ -558,15 +570,15 @@ "data": { "text/html": [ "\n", - "

method ArgoWorkflowsTriggeredRun.suspend (self)[source]

Suspend the running workflow.

Parameters
----------
**kwargs : Any
    Additional arguments to pass to the suspend command.

Returns
-------
bool
    True if the command was successful, False otherwise.

\n", + "

method ArgoWorkflowsTriggeredRun.suspend (self, **kwargs) -> bool[source]

Suspend the running workflow.

Parameters
----------
authorize : str, optional, default None
    Authorize the suspension with a production token.

Returns
-------
bool
    True if the command was successful, False otherwise.

\n", "\n", - "\n", + "\n", "\n", - "\n", + "\n", "\n", "\n", "\n", - "\t\n", + "\t\n", "\n", "\n", "\t\n", @@ -574,7 +586,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 11, @@ -591,6 +603,12 @@ "execution_count": 12, "id": "7979106a", "metadata": { + "execution": { + "iopub.execute_input": "2024-11-05T04:49:36.586491Z", + "iopub.status.busy": "2024-11-05T04:49:36.585734Z", + "iopub.status.idle": "2024-11-05T04:49:36.593263Z", + "shell.execute_reply": "2024-11-05T04:49:36.592314Z" + }, "scrolled": true }, "outputs": [ @@ -598,15 +616,15 @@ "data": { "text/html": [ "\n", - "

method ArgoWorkflowsTriggeredRun.unsuspend (self)[source]

Unsuspend the suspended workflow.

Parameters
----------
**kwargs : Any
    Additional arguments to pass to the unsuspend command.

Returns
-------
bool
    True if the command was successful, False otherwise.

\n", + "

method ArgoWorkflowsTriggeredRun.unsuspend (self, **kwargs) -> bool[source]

Unsuspend the suspended workflow.

Parameters
----------
authorize : str, optional, default None
    Authorize the unsuspend with a production token.

Returns
-------
bool
    True if the command was successful, False otherwise.

\n", "\n", - "\n", + "\n", "\n", - "\n", + "\n", "\n", "\n", "\n", - "\t\n", + "\t\n", "\n", "\n", "\t\n", @@ -614,7 +632,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 12, @@ -630,26 +648,28 @@ "cell_type": "code", "execution_count": 13, "id": "7e7a5516", - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2024-11-05T04:49:36.600298Z", + "iopub.status.busy": "2024-11-05T04:49:36.599905Z", + "iopub.status.idle": "2024-11-05T04:49:36.604958Z", + "shell.execute_reply": "2024-11-05T04:49:36.603812Z" + } + }, "outputs": [ { "data": { "text/html": [ "\n", - "

method ArgoWorkflowsTriggeredRun.status (self)[source]

Get the status of the triggered run.

Returns
-------
str, optional
    The status of the workflow considering the run object, or None if the status could not be retrieved.

\n", + "

property ArgoWorkflowsTriggeredRun.status [source]

Get the status of the triggered run.

Returns
-------
str, optional
    The status of the workflow considering the run object, or None if
    the status could not be retrieved.

\n", "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\t\n", - "\n", + "\n", + "\n", + "\n", "" ], "text/plain": [ - "" + "" ] }, "execution_count": 13, @@ -673,32 +693,46 @@ "cell_type": "code", "execution_count": 14, "id": "834cf398", - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2024-11-05T04:49:36.613501Z", + "iopub.status.busy": "2024-11-05T04:49:36.613292Z", + "iopub.status.idle": "2024-11-05T04:49:36.622167Z", + "shell.execute_reply": "2024-11-05T04:49:36.621206Z" + } + }, "outputs": [ { "data": { "text/html": [ "\n", - "

method StepFunctionsDeployer.create (self, **kwargs) -> 'StepFunctionsDeployedFlow'[source]

Create a deployed flow using the deployer implementation.

Parameters
----------
**kwargs : Any
    Additional arguments to pass to `create` corresponding to the
    command line arguments of `create`

Returns
-------
DeployedFlow
    DeployedFlow object representing the deployed flow.

Raises
------
Exception
    If there is an error during deployment.

\n", + "

method StepFunctionsDeployer.create (self, **kwargs) -> 'metaflow.plugins.aws.step_functions.step_functions_deployer_objects.StepFunctionsDeployedFlow'[source]

Create a new AWS Step Functions State Machine deployment.

Parameters
----------
authorize : str, optional, default None
    Authorize using this production token. Required when re-deploying an existing flow
    for the first time. The token is cached in METAFLOW_HOME.
generate_new_token : bool, optional, default False
    Generate a new production token for this flow. Moves the production flow to a new namespace.
given_token : str, optional, default None
    Use the given production token for this flow. Moves the production flow to the given namespace.
tags : List[str], optional, default None
    Annotate all objects produced by AWS Step Functions runs with these tags.
user_namespace : str, optional, default None
    Change the namespace from the default (production token) to the given tag.
only_json : bool, optional, default False
    Only print out JSON sent to AWS Step Functions without deploying anything.
max_workers : int, optional, default 100
    Maximum number of parallel processes.
workflow_timeout : int, optional, default None
    Workflow timeout in seconds.
log_execution_history : bool, optional, default False
    Log AWS Step Functions execution history to AWS CloudWatch Logs log group.
use_distributed_map : bool, optional, default False
    Use AWS Step Functions Distributed Map instead of Inline Map for defining foreach
    tasks in Amazon State Language.
deployer_attribute_file : str, optional, default None
    Write the workflow name to the specified file. Used internally for Metaflow's Deployer API.

Returns
-------
StepFunctionsDeployedFlow
    The Flow deployed to AWS Step Functions.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", - "\n", + "\n", "\n", - "\t\n", + "\t\n", + "\t\n", + "\t\n", + "\t\n", + "\t\n", + "\t\n", + "\t\n", + "\t\n", + "\t\n", + "\t\n", + "\t\n", "\n", "\n", - "\t\n", - "\n", - "\n", - "\t\n", + "\t\n", "\n", "" ], "text/plain": [ - "" + "" ] }, "execution_count": 14, @@ -707,51 +741,9 @@ } ], "source": [ - "from metaflow.plugins.aws.step_functions.step_functions_deployer import production_token as sfn_production_token\n", - "from metaflow.plugins.aws.step_functions.step_functions_deployer import list_runs as sfn_list_runs\n", - "from metaflow.plugins.aws.step_functions.step_functions_deployer import delete as sfn_delete\n", - "from metaflow.plugins.aws.step_functions.step_functions_deployer import trigger as sfn_trigger\n", - "from metaflow.plugins.aws.step_functions.step_functions_deployer import terminate as sfn_terminate\n", - "\n", - "class StepFunctionsTriggeredRun:\n", - " \n", - " @property\n", - " def run(self):\n", - " pass\n", - " \n", - " def terminate(self):\n", - " pass\n", - " \n", - "\n", - "StepFunctionsTriggeredRun.__doc__ = TriggeredRun.__doc__\n", - "StepFunctionsTriggeredRun.run.__doc__ = TriggeredRun.run.__doc__\n", - "StepFunctionsTriggeredRun.terminate.__doc__ = sfn_terminate.__doc__\n", - "\n", - "\n", - "class StepFunctionsDeployedFlow:\n", - " @property\n", - " def production_token(self):\n", - " pass\n", - " \n", - " def trigger(self) -> 'StepFunctionsTriggeredRun':\n", - " pass\n", - " \n", - " def delete(self):\n", - " pass\n", - "\n", - " def list_runs(self, states):\n", - " pass\n", - " \n", - "StepFunctionsDeployedFlow.production_token.__doc__ = sfn_production_token.__doc__\n", - "StepFunctionsDeployedFlow.trigger.__doc__ = sfn_trigger.__doc__\n", - "StepFunctionsDeployedFlow.delete.__doc__ = sfn_delete.__doc__\n", - "StepFunctionsDeployedFlow.list_runs.__doc__ = sfn_list_runs.__doc__\n", - "\n", - "class StepFunctionsDeployer:\n", - " def create(self, **kwargs) -> 'StepFunctionsDeployedFlow':\n", - " pass\n", - "\n", - "StepFunctionsDeployer.create.__doc__ = DeployerImpl.create.__doc__\n", + "from metaflow.plugins.aws.step_functions.step_functions_deployer_objects import StepFunctionsTriggeredRun\n", + "from metaflow.plugins.aws.step_functions.step_functions_deployer_objects import StepFunctionsDeployedFlow\n", + "from metaflow.plugins.aws.step_functions.step_functions_deployer import StepFunctionsDeployer\n", "\n", "ShowDoc(StepFunctionsDeployer.create)" ] @@ -768,15 +760,22 @@ "cell_type": "code", "execution_count": 15, "id": "0dddad99", - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2024-11-05T04:49:36.627329Z", + "iopub.status.busy": "2024-11-05T04:49:36.626963Z", + "iopub.status.idle": "2024-11-05T04:49:36.631407Z", + "shell.execute_reply": "2024-11-05T04:49:36.630594Z" + } + }, "outputs": [ { "data": { "text/html": [ "\n", - "

property StepFunctionsDeployedFlow.production_token

Get the production token for the deployed flow.

Returns
-------
str, optional
    The production token, None if it cannot be retrieved.

\n", + "

property StepFunctionsDeployedFlow.production_token [source]

Get the production token for the deployed flow.

Returns
-------
str, optional
    The production token, None if it cannot be retrieved.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -785,7 +784,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 15, @@ -801,21 +800,28 @@ "cell_type": "code", "execution_count": 16, "id": "4fa049eb", - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2024-11-05T04:49:36.640265Z", + "iopub.status.busy": "2024-11-05T04:49:36.639734Z", + "iopub.status.idle": "2024-11-05T04:49:36.647171Z", + "shell.execute_reply": "2024-11-05T04:49:36.646306Z" + } + }, "outputs": [ { "data": { "text/html": [ "\n", - "

method StepFunctionsDeployedFlow.trigger (self) -> 'StepFunctionsTriggeredRun'[source]

Trigger a new run for the deployed flow.

Parameters
----------
**kwargs : Any
    Additional arguments to pass to the trigger command, `Parameters` in particular

Returns
-------
StepFunctionsTriggeredRun
    The triggered run instance.

Raises
------
Exception
    If there is an error during the trigger process.

\n", + "

method StepFunctionsDeployedFlow.trigger (self, **kwargs) -> metaflow.plugins.aws.step_functions.step_functions_deployer_objects.StepFunctionsTriggeredRun[source]

Trigger a new run for the deployed flow.

Parameters
----------
**kwargs : Any
    Additional arguments to pass to the trigger command,
    `Parameters` in particular

Returns
-------
StepFunctionsTriggeredRun
    The triggered run instance.

Raises
------
Exception
    If there is an error during the trigger process.

\n", "\n", - "\n", + "\n", "\n", - "\n", + "\n", "\n", "\n", "\n", - "\t\n", + "\t\n", "\n", "\n", "\t\n", @@ -826,7 +832,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 16, @@ -842,21 +848,28 @@ "cell_type": "code", "execution_count": 17, "id": "f4f2d783", - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2024-11-05T04:49:36.655817Z", + "iopub.status.busy": "2024-11-05T04:49:36.655552Z", + "iopub.status.idle": "2024-11-05T04:49:36.660966Z", + "shell.execute_reply": "2024-11-05T04:49:36.659976Z" + } + }, "outputs": [ { "data": { "text/html": [ "\n", - "

method StepFunctionsDeployedFlow.delete (self)[source]

Delete the deployed flow.

Parameters
----------
**kwargs : Any
    Additional arguments to pass to the delete command.

Returns
-------
bool
    True if the command was successful, False otherwise.

\n", + "

method StepFunctionsDeployedFlow.delete (self, **kwargs) -> bool[source]

Delete the deployed state machine.

Parameters
----------
authorize : str, optional, default None
    Authorize the deletion with a production token.

Returns
-------
bool
    True if the command was successful, False otherwise.

\n", "\n", - "\n", + "\n", "\n", - "\n", + "\n", "\n", - "\n", + "\n", "\n", - "\t\n", + "\t\n", "\n", "\n", "\t\n", @@ -864,7 +877,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 17, @@ -880,24 +893,31 @@ "cell_type": "code", "execution_count": 18, "id": "a88c5e2d", - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2024-11-05T04:49:36.668385Z", + "iopub.status.busy": "2024-11-05T04:49:36.668026Z", + "iopub.status.idle": "2024-11-05T04:49:36.674199Z", + "shell.execute_reply": "2024-11-05T04:49:36.673463Z" + } + }, "outputs": [ { "data": { "text/html": [ "\n", - "

method StepFunctionsDeployedFlow.list_runs (self, states)[source]

List runs of the deployed flow.

Parameters
----------
states : Optional[List[str]], optional
    A list of states to filter the runs by. Allowed values are:
    RUNNING, SUCCEEDED, FAILED, TIMED_OUT, ABORTED.
    If not provided, all states will be considered.

Returns
-------
List[TriggeredRun]
    A list of TriggeredRun objects representing the runs of the deployed flow.

Raises
------
ValueError
    If any of the provided states are invalid or if there are duplicate states.

\n", + "

method StepFunctionsDeployedFlow.list_runs (self, states: Optional[List[str]] = None) -> List[metaflow.plugins.aws.step_functions.step_functions_deployer_objects.StepFunctionsTriggeredRun][source]

List runs of the deployed flow.

Parameters
----------
states : List[str], optional, default None
    A list of states to filter the runs by. Allowed values are:
    RUNNING, SUCCEEDED, FAILED, TIMED_OUT, ABORTED.
    If not provided, all states will be considered.

Returns
-------
List[StepFunctionsTriggeredRun]
    A list of TriggeredRun objects representing the runs of the deployed flow.

Raises
------
ValueError
    If any of the provided states are invalid or if there are duplicate states.

\n", "\n", - "\n", + "\n", "\n", - "\n", + "\n", "\n", "\n", "\n", - "\t\n", + "\t\n", "\n", "\n", - "\t\n", + "\t\n", "\n", "\n", "\t\n", @@ -905,7 +925,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 18, @@ -929,15 +949,22 @@ "cell_type": "code", "execution_count": 19, "id": "a4ab8559", - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2024-11-05T04:49:36.683686Z", + "iopub.status.busy": "2024-11-05T04:49:36.683370Z", + "iopub.status.idle": "2024-11-05T04:49:36.689699Z", + "shell.execute_reply": "2024-11-05T04:49:36.687674Z" + } + }, "outputs": [ { "data": { "text/html": [ "\n", - "

property StepFunctionsTriggeredRun.run

Retrieve the `Run` object for the triggered run.

Note that Metaflow `Run` becomes available only when the `start` task
has started executing.

Returns
-------
Run, optional
    Metaflow Run object if the `start` step has started executing, otherwise None.

\n", + "

property TriggeredRun.run [source]

Retrieve the `Run` object for the triggered run.

Note that Metaflow `Run` becomes available only when the `start` task
has started executing.

Returns
-------
Run, optional
    Metaflow Run object if the `start` step has started executing, otherwise None.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -946,7 +973,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 19, @@ -962,21 +989,28 @@ "cell_type": "code", "execution_count": 20, "id": "5e2ec6b7", - "metadata": {}, + "metadata": { + "execution": { + "iopub.execute_input": "2024-11-05T04:49:36.697346Z", + "iopub.status.busy": "2024-11-05T04:49:36.697097Z", + "iopub.status.idle": "2024-11-05T04:49:36.704126Z", + "shell.execute_reply": "2024-11-05T04:49:36.702806Z" + } + }, "outputs": [ { "data": { "text/html": [ "\n", - "

method StepFunctionsTriggeredRun.terminate (self)[source]

Terminate the running workflow.

Parameters
----------
**kwargs : Any
    Additional arguments to pass to the terminate command.

Returns
-------
bool
    True if the command was successful, False otherwise.

\n", + "

method StepFunctionsTriggeredRun.terminate (self, **kwargs) -> bool[source]

Terminate the running state machine execution.

Parameters
----------
authorize : str, optional, default None
    Authorize the termination with a production token.

Returns
-------
bool
    True if the command was successful, False otherwise.

\n", "\n", - "\n", + "\n", "\n", - "\n", + "\n", "\n", - "\n", + "\n", "\n", - "\t\n", + "\t\n", "\n", "\n", "\t\n", @@ -984,7 +1018,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 20, @@ -1021,7 +1055,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0" + "version": "3.10.13" } }, "nbformat": 4, diff --git a/docs/api/deployer.md b/docs/api/deployer.md index dcf276f0..677a7396 100644 --- a/docs/api/deployer.md +++ b/docs/api/deployer.md @@ -48,6 +48,7 @@ In addition to this basic functionality, each implementation-specific object exp + @@ -80,19 +81,32 @@ In addition to this basic functionality, each implementation-specific object exp ## Deploy Argo Workflows with `ArgoWorkflowsDeployer` - + - + - + + + + + + + + + + + + + + + + + - - - - + @@ -100,7 +114,7 @@ In addition to this basic functionality, each implementation-specific object exp ### Manage a flow deployed on Argo Workflows with `ArgoWorkflowsDeployedFlow` - + @@ -110,13 +124,13 @@ In addition to this basic functionality, each implementation-specific object exp - + - + - + @@ -128,13 +142,13 @@ In addition to this basic functionality, each implementation-specific object exp - + - + - + - + @@ -145,7 +159,7 @@ In addition to this basic functionality, each implementation-specific object exp ### Manage a run triggered on Argo Workflows with `ArgoWorkflowsTriggeredRun` - + @@ -155,13 +169,13 @@ In addition to this basic functionality, each implementation-specific object exp - + - + - + @@ -170,13 +184,13 @@ In addition to this basic functionality, each implementation-specific object exp - + - + - + @@ -185,13 +199,13 @@ In addition to this basic functionality, each implementation-specific object exp - + - + - + @@ -200,33 +214,35 @@ In addition to this basic functionality, each implementation-specific object exp - - - - - - - - + + + ## Deploy Step Functions with `StepFunctionsDeployer` - + - + - + + + + + + + + + + + - - - - + @@ -234,7 +250,7 @@ In addition to this basic functionality, each implementation-specific object exp ### Manage a flow deployed on Step Functions with `StepFunctionsDeployedFlow` - + @@ -244,13 +260,13 @@ In addition to this basic functionality, each implementation-specific object exp - + - + - + @@ -262,13 +278,13 @@ In addition to this basic functionality, each implementation-specific object exp - + - + - + - + @@ -277,16 +293,16 @@ In addition to this basic functionality, each implementation-specific object exp - + - + - + - + @@ -297,7 +313,7 @@ In addition to this basic functionality, each implementation-specific object exp ### Manage a run triggered on Step Functions with `StepFunctionsTriggeredRun` - + @@ -307,13 +323,13 @@ In addition to this basic functionality, each implementation-specific object exp - + - + - + - + diff --git a/docs/api/flow-decorators/conda_base.ipynb b/docs/api/flow-decorators/conda_base.ipynb index 7f0fc10e..7fc3ae9a 100644 --- a/docs/api/flow-decorators/conda_base.ipynb +++ b/docs/api/flow-decorators/conda_base.ipynb @@ -18,10 +18,10 @@ "id": "8d5bb116", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:17:04.514901Z", - "iopub.status.busy": "2024-07-25T06:17:04.514765Z", - "iopub.status.idle": "2024-07-25T06:17:04.822231Z", - "shell.execute_reply": "2024-07-25T06:17:04.821959Z" + "iopub.execute_input": "2024-11-05T04:49:45.632413Z", + "iopub.status.busy": "2024-11-05T04:49:45.632293Z", + "iopub.status.idle": "2024-11-05T04:49:45.928351Z", + "shell.execute_reply": "2024-11-05T04:49:45.928001Z" } }, "outputs": [], @@ -40,10 +40,10 @@ "id": "29af6ee3", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:17:04.824518Z", - "iopub.status.busy": "2024-07-25T06:17:04.824364Z", - "iopub.status.idle": "2024-07-25T06:17:04.831032Z", - "shell.execute_reply": "2024-07-25T06:17:04.830771Z" + "iopub.execute_input": "2024-11-05T04:49:45.931166Z", + "iopub.status.busy": "2024-11-05T04:49:45.930997Z", + "iopub.status.idle": "2024-11-05T04:49:45.938528Z", + "shell.execute_reply": "2024-11-05T04:49:45.938272Z" } }, "outputs": [ @@ -51,9 +51,9 @@ "data": { "text/html": [ "\n", - "

decorator @conda_base (...)[source]

metaflow

Specifies the Conda environment for all steps of the flow.

Use `@conda_base` to set common libraries required by all
steps and use `@conda` to specify step-specific additions.

Parameters
----------
packages : Dict[str, str], default {}
    Packages to use for this flow. The key is the name of the package
    and the value is the version to use.
libraries : Dict[str, str], default {}
    Supported for backward compatibility. When used with packages, packages will take precedence.
python : str, optional, default None
    Version of Python to use, e.g. '3.7.4'. A default value of None implies
    that the version used will correspond to the version of the Python interpreter used to start the run.
disabled : bool, default False
    If set to True, disables Conda.

\n", + "

decorator @conda_base (...)[source]

metaflow

Specifies the Conda environment for all steps of the flow.

Use `@conda_base` to set common libraries required by all
steps and use `@conda` to specify step-specific additions.

Parameters
----------
packages : Dict[str, str], default {}
    Packages to use for this flow. The key is the name of the package
    and the value is the version to use.
libraries : Dict[str, str], default {}
    Supported for backward compatibility. When used with packages, packages will take precedence.
python : str, optional, default None
    Version of Python to use, e.g. '3.7.4'. A default value of None implies
    that the version used will correspond to the version of the Python interpreter used to start the run.
disabled : bool, default False
    If set to True, disables Conda.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -67,7 +67,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -104,7 +104,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0" + "version": "3.10.13" } }, "nbformat": 4, diff --git a/docs/api/flow-decorators/conda_base.md b/docs/api/flow-decorators/conda_base.md index fbce3bef..652fb5f5 100644 --- a/docs/api/flow-decorators/conda_base.md +++ b/docs/api/flow-decorators/conda_base.md @@ -7,7 +7,7 @@ The libraries are installed from [Conda repositories](https://anaconda.org/). Fo - + diff --git a/docs/api/flow-decorators/project.ipynb b/docs/api/flow-decorators/project.ipynb index bc7201a0..7e687bae 100644 --- a/docs/api/flow-decorators/project.ipynb +++ b/docs/api/flow-decorators/project.ipynb @@ -22,10 +22,10 @@ "id": "8d5bb116", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:17:05.193031Z", - "iopub.status.busy": "2024-07-25T06:17:05.192942Z", - "iopub.status.idle": "2024-07-25T06:17:05.451383Z", - "shell.execute_reply": "2024-07-25T06:17:05.451056Z" + "iopub.execute_input": "2024-11-05T04:49:46.390007Z", + "iopub.status.busy": "2024-11-05T04:49:46.389914Z", + "iopub.status.idle": "2024-11-05T04:49:46.663594Z", + "shell.execute_reply": "2024-11-05T04:49:46.663242Z" } }, "outputs": [], @@ -44,10 +44,10 @@ "id": "29af6ee3", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:17:05.453564Z", - "iopub.status.busy": "2024-07-25T06:17:05.453413Z", - "iopub.status.idle": "2024-07-25T06:17:05.460020Z", - "shell.execute_reply": "2024-07-25T06:17:05.459767Z" + "iopub.execute_input": "2024-11-05T04:49:46.666431Z", + "iopub.status.busy": "2024-11-05T04:49:46.666236Z", + "iopub.status.idle": "2024-11-05T04:49:46.673486Z", + "shell.execute_reply": "2024-11-05T04:49:46.673181Z" } }, "outputs": [ @@ -55,13 +55,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "/Users/ville/mambaforge/envs/docs/lib/python3.11/site-packages/numpydoc/docscrape.py:434: UserWarning: potentially wrong underline length... \n", + "/Users/rcledat/mambaforge/envs/metaflow-oss-docs/lib/python3.10/site-packages/numpydoc/docscrape.py:434: UserWarning: potentially wrong underline length... \n", "@@ Returns \n", "------- in \n", "Specifies what flows belong to the same project.\n", - "... in the docstring of ProjectDecorator in /Users/ville/src/madhur-metaflow/metaflow/plugins/project_decorator.py.\n", + "... in the docstring of ProjectDecorator in /Users/rcledat/mambaforge/envs/metaflow-oss-docs/lib/python3.10/site-packages/metaflow/plugins/project_decorator.py.\n", " warn(msg)\n", - "/Users/ville/mambaforge/envs/docs/lib/python3.11/site-packages/numpydoc/docscrape.py:434: UserWarning: Unknown section Mf Add To Current in the docstring of ProjectDecorator in /Users/ville/src/madhur-metaflow/metaflow/plugins/project_decorator.py.\n", + "/Users/rcledat/mambaforge/envs/metaflow-oss-docs/lib/python3.10/site-packages/numpydoc/docscrape.py:434: UserWarning: Unknown section Mf Add To Current in the docstring of ProjectDecorator in /Users/rcledat/mambaforge/envs/metaflow-oss-docs/lib/python3.10/site-packages/metaflow/plugins/project_decorator.py.\n", " warn(msg)\n" ] }, @@ -82,7 +82,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -119,7 +119,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0" + "version": "3.10.13" } }, "nbformat": 4, diff --git a/docs/api/flow-decorators/schedule.ipynb b/docs/api/flow-decorators/schedule.ipynb index 38f368f5..68770078 100644 --- a/docs/api/flow-decorators/schedule.ipynb +++ b/docs/api/flow-decorators/schedule.ipynb @@ -18,10 +18,10 @@ "id": "8d5bb116", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:17:05.741334Z", - "iopub.status.busy": "2024-07-25T06:17:05.741233Z", - "iopub.status.idle": "2024-07-25T06:17:05.999865Z", - "shell.execute_reply": "2024-07-25T06:17:05.999582Z" + "iopub.execute_input": "2024-11-05T04:49:46.685891Z", + "iopub.status.busy": "2024-11-05T04:49:46.685792Z", + "iopub.status.idle": "2024-11-05T04:49:46.973883Z", + "shell.execute_reply": "2024-11-05T04:49:46.973485Z" } }, "outputs": [], @@ -40,10 +40,10 @@ "id": "29af6ee3", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:17:06.002165Z", - "iopub.status.busy": "2024-07-25T06:17:06.002025Z", - "iopub.status.idle": "2024-07-25T06:17:06.007063Z", - "shell.execute_reply": "2024-07-25T06:17:06.006812Z" + "iopub.execute_input": "2024-11-05T04:49:46.976946Z", + "iopub.status.busy": "2024-11-05T04:49:46.976702Z", + "iopub.status.idle": "2024-11-05T04:49:46.983011Z", + "shell.execute_reply": "2024-11-05T04:49:46.982758Z" } }, "outputs": [ @@ -68,7 +68,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -105,7 +105,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0" + "version": "3.10.13" } }, "nbformat": 4, diff --git a/docs/api/flow-decorators/trigger.ipynb b/docs/api/flow-decorators/trigger.ipynb index 42e6c8b1..b2673f99 100644 --- a/docs/api/flow-decorators/trigger.ipynb +++ b/docs/api/flow-decorators/trigger.ipynb @@ -18,10 +18,10 @@ "id": "8d5bb116", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:17:06.204233Z", - "iopub.status.busy": "2024-07-25T06:17:06.204133Z", - "iopub.status.idle": "2024-07-25T06:17:06.488531Z", - "shell.execute_reply": "2024-07-25T06:17:06.488256Z" + "iopub.execute_input": "2024-11-05T04:49:47.133813Z", + "iopub.status.busy": "2024-11-05T04:49:47.133719Z", + "iopub.status.idle": "2024-11-05T04:49:47.416266Z", + "shell.execute_reply": "2024-11-05T04:49:47.415888Z" } }, "outputs": [], @@ -40,10 +40,10 @@ "id": "29af6ee3", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:17:06.490772Z", - "iopub.status.busy": "2024-07-25T06:17:06.490640Z", - "iopub.status.idle": "2024-07-25T06:17:06.497814Z", - "shell.execute_reply": "2024-07-25T06:17:06.497586Z" + "iopub.execute_input": "2024-11-05T04:49:47.419294Z", + "iopub.status.busy": "2024-11-05T04:49:47.419057Z", + "iopub.status.idle": "2024-11-05T04:49:47.427768Z", + "shell.execute_reply": "2024-11-05T04:49:47.427460Z" } }, "outputs": [ @@ -51,13 +51,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "/Users/ville/mambaforge/envs/docs/lib/python3.11/site-packages/numpydoc/docscrape.py:434: UserWarning: potentially wrong underline length... \n", + "/Users/rcledat/mambaforge/envs/metaflow-oss-docs/lib/python3.10/site-packages/numpydoc/docscrape.py:434: UserWarning: potentially wrong underline length... \n", "@@ Returns \n", "------- in \n", "Specifies the event(s) that this flow depends on.\n", - "... in the docstring of TriggerDecorator in /Users/ville/src/madhur-metaflow/metaflow/plugins/events_decorator.py.\n", + "... in the docstring of TriggerDecorator in /Users/rcledat/mambaforge/envs/metaflow-oss-docs/lib/python3.10/site-packages/metaflow/plugins/events_decorator.py.\n", " warn(msg)\n", - "/Users/ville/mambaforge/envs/docs/lib/python3.11/site-packages/numpydoc/docscrape.py:434: UserWarning: Unknown section Mf Add To Current in the docstring of TriggerDecorator in /Users/ville/src/madhur-metaflow/metaflow/plugins/events_decorator.py.\n", + "/Users/rcledat/mambaforge/envs/metaflow-oss-docs/lib/python3.10/site-packages/numpydoc/docscrape.py:434: UserWarning: Unknown section Mf Add To Current in the docstring of TriggerDecorator in /Users/rcledat/mambaforge/envs/metaflow-oss-docs/lib/python3.10/site-packages/metaflow/plugins/events_decorator.py.\n", " warn(msg)\n" ] }, @@ -80,7 +80,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -117,7 +117,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0" + "version": "3.10.13" } }, "nbformat": 4, diff --git a/docs/api/flow-decorators/trigger_on_finish.ipynb b/docs/api/flow-decorators/trigger_on_finish.ipynb index 2af718b6..8d4fd6d6 100644 --- a/docs/api/flow-decorators/trigger_on_finish.ipynb +++ b/docs/api/flow-decorators/trigger_on_finish.ipynb @@ -18,10 +18,10 @@ "id": "8d5bb116", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:17:06.568582Z", - "iopub.status.busy": "2024-07-25T06:17:06.568499Z", - "iopub.status.idle": "2024-07-25T06:17:06.834174Z", - "shell.execute_reply": "2024-07-25T06:17:06.833777Z" + "iopub.execute_input": "2024-11-05T04:49:47.847964Z", + "iopub.status.busy": "2024-11-05T04:49:47.847436Z", + "iopub.status.idle": "2024-11-05T04:49:48.140357Z", + "shell.execute_reply": "2024-11-05T04:49:48.139990Z" } }, "outputs": [], @@ -40,10 +40,10 @@ "id": "29af6ee3", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:17:06.836450Z", - "iopub.status.busy": "2024-07-25T06:17:06.836311Z", - "iopub.status.idle": "2024-07-25T06:17:06.843909Z", - "shell.execute_reply": "2024-07-25T06:17:06.843568Z" + "iopub.execute_input": "2024-11-05T04:49:48.143639Z", + "iopub.status.busy": "2024-11-05T04:49:48.143434Z", + "iopub.status.idle": "2024-11-05T04:49:48.152821Z", + "shell.execute_reply": "2024-11-05T04:49:48.152521Z" } }, "outputs": [ @@ -51,13 +51,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "/Users/ville/mambaforge/envs/docs/lib/python3.11/site-packages/numpydoc/docscrape.py:434: UserWarning: potentially wrong underline length... \n", + "/Users/rcledat/mambaforge/envs/metaflow-oss-docs/lib/python3.10/site-packages/numpydoc/docscrape.py:434: UserWarning: potentially wrong underline length... \n", "@@ Returns \n", "------- in \n", "Specifies the flow(s) that this flow depends on.\n", - "... in the docstring of TriggerOnFinishDecorator in /Users/ville/src/madhur-metaflow/metaflow/plugins/events_decorator.py.\n", + "... in the docstring of TriggerOnFinishDecorator in /Users/rcledat/mambaforge/envs/metaflow-oss-docs/lib/python3.10/site-packages/metaflow/plugins/events_decorator.py.\n", " warn(msg)\n", - "/Users/ville/mambaforge/envs/docs/lib/python3.11/site-packages/numpydoc/docscrape.py:434: UserWarning: Unknown section Mf Add To Current in the docstring of TriggerOnFinishDecorator in /Users/ville/src/madhur-metaflow/metaflow/plugins/events_decorator.py.\n", + "/Users/rcledat/mambaforge/envs/metaflow-oss-docs/lib/python3.10/site-packages/numpydoc/docscrape.py:434: UserWarning: Unknown section Mf Add To Current in the docstring of TriggerOnFinishDecorator in /Users/rcledat/mambaforge/envs/metaflow-oss-docs/lib/python3.10/site-packages/metaflow/plugins/events_decorator.py.\n", " warn(msg)\n" ] }, @@ -80,7 +80,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -117,7 +117,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0" + "version": "3.10.13" } }, "nbformat": 4, diff --git a/docs/api/flowspec.ipynb b/docs/api/flowspec.ipynb index b0511480..3fdb0c93 100644 --- a/docs/api/flowspec.ipynb +++ b/docs/api/flowspec.ipynb @@ -46,10 +46,10 @@ "id": "a5ef9454", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:58.863742Z", - "iopub.status.busy": "2024-07-25T06:16:58.863656Z", - "iopub.status.idle": "2024-07-25T06:16:59.136317Z", - "shell.execute_reply": "2024-07-25T06:16:59.136010Z" + "iopub.execute_input": "2024-11-05T04:49:39.993012Z", + "iopub.status.busy": "2024-11-05T04:49:39.992831Z", + "iopub.status.idle": "2024-11-05T04:49:40.297574Z", + "shell.execute_reply": "2024-11-05T04:49:40.297246Z" } }, "outputs": [ @@ -96,10 +96,10 @@ "id": "5ff62112", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:59.138672Z", - "iopub.status.busy": "2024-07-25T06:16:59.138509Z", - "iopub.status.idle": "2024-07-25T06:16:59.143884Z", - "shell.execute_reply": "2024-07-25T06:16:59.143633Z" + "iopub.execute_input": "2024-11-05T04:49:40.300873Z", + "iopub.status.busy": "2024-11-05T04:49:40.300661Z", + "iopub.status.idle": "2024-11-05T04:49:40.306595Z", + "shell.execute_reply": "2024-11-05T04:49:40.306262Z" } }, "outputs": [ @@ -107,9 +107,9 @@ "data": { "text/html": [ "\n", - "

method FlowSpec.next *dsts, foreach=None[source]

Indicates the next step to execute after this step has completed.

This statement should appear as the last statement of each step, except
the end step.

There are several valid formats to specify the next step:

- Straight-line connection: `self.next(self.next_step)` where `next_step` is a method in
  the current class decorated with the `@step` decorator.

- Static fan-out connection: `self.next(self.step1, self.step2, ...)` where `stepX` are
  methods in the current class decorated with the `@step` decorator.

- Foreach branch:
  ```
  self.next(self.foreach_step, foreach='foreach_iterator')
  ```
  In this situation, `foreach_step` is a method in the current class decorated with the
  `@step` decorator and `foreach_iterator` is a variable name in the current class that
  evaluates to an iterator. A task will be launched for each value in the iterator and
  each task will execute the code specified by the step `foreach_step`.

Parameters
----------
dsts : Callable[..., None]
    One or more methods annotated with `@step`.

Raises
------
InvalidNextException
    Raised if the format of the arguments does not match one of the ones given above.

\n", + "

method FlowSpec.next *dsts, foreach=None[source]

Indicates the next step to execute after this step has completed.

This statement should appear as the last statement of each step, except
the end step.

There are several valid formats to specify the next step:

- Straight-line connection: `self.next(self.next_step)` where `next_step` is a method in
  the current class decorated with the `@step` decorator.

- Static fan-out connection: `self.next(self.step1, self.step2, ...)` where `stepX` are
  methods in the current class decorated with the `@step` decorator.

- Foreach branch:
  ```
  self.next(self.foreach_step, foreach='foreach_iterator')
  ```
  In this situation, `foreach_step` is a method in the current class decorated with the
  `@step` decorator and `foreach_iterator` is a variable name in the current class that
  evaluates to an iterator. A task will be launched for each value in the iterator and
  each task will execute the code specified by the step `foreach_step`.

Parameters
----------
dsts : Callable[..., None]
    One or more methods annotated with `@step`.

Raises
------
InvalidNextException
    Raised if the format of the arguments does not match one of the ones given above.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -123,7 +123,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -151,10 +151,10 @@ "id": "64fcbf33", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:59.145679Z", - "iopub.status.busy": "2024-07-25T06:16:59.145585Z", - "iopub.status.idle": "2024-07-25T06:16:59.147692Z", - "shell.execute_reply": "2024-07-25T06:16:59.147438Z" + "iopub.execute_input": "2024-11-05T04:49:40.309426Z", + "iopub.status.busy": "2024-11-05T04:49:40.309284Z", + "iopub.status.idle": "2024-11-05T04:49:40.311666Z", + "shell.execute_reply": "2024-11-05T04:49:40.311362Z" } }, "outputs": [ @@ -173,7 +173,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 3, @@ -191,10 +191,10 @@ "id": "13dacb8d", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:59.149936Z", - "iopub.status.busy": "2024-07-25T06:16:59.149848Z", - "iopub.status.idle": "2024-07-25T06:16:59.152064Z", - "shell.execute_reply": "2024-07-25T06:16:59.151791Z" + "iopub.execute_input": "2024-11-05T04:49:40.316437Z", + "iopub.status.busy": "2024-11-05T04:49:40.316303Z", + "iopub.status.idle": "2024-11-05T04:49:40.318852Z", + "shell.execute_reply": "2024-11-05T04:49:40.318499Z" } }, "outputs": [ @@ -213,7 +213,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 4, @@ -231,10 +231,10 @@ "id": "684b4601", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:59.154283Z", - "iopub.status.busy": "2024-07-25T06:16:59.154195Z", - "iopub.status.idle": "2024-07-25T06:16:59.156811Z", - "shell.execute_reply": "2024-07-25T06:16:59.156579Z" + "iopub.execute_input": "2024-11-05T04:49:40.322027Z", + "iopub.status.busy": "2024-11-05T04:49:40.321826Z", + "iopub.status.idle": "2024-11-05T04:49:40.325179Z", + "shell.execute_reply": "2024-11-05T04:49:40.324893Z" } }, "outputs": [ @@ -242,9 +242,9 @@ "data": { "text/html": [ "\n", - "

method FlowSpec.foreach_stack (self) -> Optional[List[Tuple[int, int, Any]]][source]

Returns the current stack of foreach indexes and values for the current step.

Use this information to understand what data is being processed in the current
foreach branch. For example, considering the following code:
```
@step
def root(self):
    self.split_1 = ['a', 'b', 'c']
    self.next(self.nest_1, foreach='split_1')

@step
def nest_1(self):
    self.split_2 = ['d', 'e', 'f', 'g']
    self.next(self.nest_2, foreach='split_2'):

@step
def nest_2(self):
    foo = self.foreach_stack()
```

`foo` will take the following values in the various tasks for nest_2:
```
    [(0, 3, 'a'), (0, 4, 'd')]
    [(0, 3, 'a'), (1, 4, 'e')]
    ...
    [(0, 3, 'a'), (3, 4, 'g')]
    [(1, 3, 'b'), (0, 4, 'd')]
    ...
```
where each tuple corresponds to:

- The index of the task for that level of the loop.
- The number of splits for that level of the loop.
- The value for that level of the loop.

Note that the last tuple returned in a task corresponds to:

- 1st element: value returned by `self.index`.
- 3rd element: value returned by `self.input`.

Returns
-------
List[Tuple[int, int, Any]]
    An array describing the current stack of foreach steps.

\n", + "

method FlowSpec.foreach_stack (self) -> Optional[List[Tuple[int, int, Any]]][source]

Returns the current stack of foreach indexes and values for the current step.

Use this information to understand what data is being processed in the current
foreach branch. For example, considering the following code:
```
@step
def root(self):
    self.split_1 = ['a', 'b', 'c']
    self.next(self.nest_1, foreach='split_1')

@step
def nest_1(self):
    self.split_2 = ['d', 'e', 'f', 'g']
    self.next(self.nest_2, foreach='split_2'):

@step
def nest_2(self):
    foo = self.foreach_stack()
```

`foo` will take the following values in the various tasks for nest_2:
```
    [(0, 3, 'a'), (0, 4, 'd')]
    [(0, 3, 'a'), (1, 4, 'e')]
    ...
    [(0, 3, 'a'), (3, 4, 'g')]
    [(1, 3, 'b'), (0, 4, 'd')]
    ...
```
where each tuple corresponds to:

- The index of the task for that level of the loop.
- The number of splits for that level of the loop.
- The value for that level of the loop.

Note that the last tuple returned in a task corresponds to:

- 1st element: value returned by `self.index`.
- 3rd element: value returned by `self.input`.

Returns
-------
List[Tuple[int, int, Any]]
    An array describing the current stack of foreach steps.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -255,7 +255,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 5, @@ -273,10 +273,10 @@ "id": "e45999eb", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:59.159193Z", - "iopub.status.busy": "2024-07-25T06:16:59.159095Z", - "iopub.status.idle": "2024-07-25T06:16:59.162186Z", - "shell.execute_reply": "2024-07-25T06:16:59.161965Z" + "iopub.execute_input": "2024-11-05T04:49:40.327905Z", + "iopub.status.busy": "2024-11-05T04:49:40.327808Z", + "iopub.status.idle": "2024-11-05T04:49:40.331552Z", + "shell.execute_reply": "2024-11-05T04:49:40.331158Z" } }, "outputs": [ @@ -284,9 +284,9 @@ "data": { "text/html": [ "\n", - "

method FlowSpec.merge_artifacts (self, inputs: metaflow.datastore.inputs.Inputs, exclude: Optional[List[str]] = None, include: Optional[List[str]] = None) -> None[source]

Helper function for merging artifacts in a join step.

This function takes all the artifacts coming from the branches of a
join point and assigns them to self in the calling step. Only artifacts
not set in the current step are considered. If, for a given artifact, different
values are present on the incoming edges, an error will be thrown and the artifacts
that conflict will be reported.

As a few examples, in the simple graph: A splitting into B and C and joining in D:
```
A:
  self.x = 5
  self.y = 6
B:
  self.b_var = 1
  self.x = from_b
C:
  self.x = from_c

D:
  merge_artifacts(inputs)
```
In D, the following artifacts are set:
  - `y` (value: 6), `b_var` (value: 1)
  - if `from_b` and `from_c` are the same, `x` will be accessible and have value `from_b`
  - if `from_b` and `from_c` are different, an error will be thrown. To prevent this error,
    you need to manually set `self.x` in D to a merged value (for example the max) prior to
    calling `merge_artifacts`.

Parameters
----------
inputs : Inputs
    Incoming steps to the join point.
exclude : List[str], optional, default None
    If specified, do not consider merging artifacts with a name in `exclude`.
    Cannot specify if `include` is also specified.
include : List[str], optional, default None
    If specified, only merge artifacts specified. Cannot specify if `exclude` is
    also specified.

Raises
------
MetaflowException
    This exception is thrown if this is not called in a join step.
UnhandledInMergeArtifactsException
    This exception is thrown in case of unresolved conflicts.
MissingInMergeArtifactsException
    This exception is thrown in case an artifact specified in `include` cannot
    be found.

\n", + "

method FlowSpec.merge_artifacts (self, inputs: metaflow.datastore.inputs.Inputs, exclude: Optional[List[str]] = None, include: Optional[List[str]] = None) -> None[source]

Helper function for merging artifacts in a join step.

This function takes all the artifacts coming from the branches of a
join point and assigns them to self in the calling step. Only artifacts
not set in the current step are considered. If, for a given artifact, different
values are present on the incoming edges, an error will be thrown and the artifacts
that conflict will be reported.

As a few examples, in the simple graph: A splitting into B and C and joining in D:
```
A:
  self.x = 5
  self.y = 6
B:
  self.b_var = 1
  self.x = from_b
C:
  self.x = from_c

D:
  merge_artifacts(inputs)
```
In D, the following artifacts are set:
  - `y` (value: 6), `b_var` (value: 1)
  - if `from_b` and `from_c` are the same, `x` will be accessible and have value `from_b`
  - if `from_b` and `from_c` are different, an error will be thrown. To prevent this error,
    you need to manually set `self.x` in D to a merged value (for example the max) prior to
    calling `merge_artifacts`.

Parameters
----------
inputs : Inputs
    Incoming steps to the join point.
exclude : List[str], optional, default None
    If specified, do not consider merging artifacts with a name in `exclude`.
    Cannot specify if `include` is also specified.
include : List[str], optional, default None
    If specified, only merge artifacts specified. Cannot specify if `exclude` is
    also specified.

Raises
------
MetaflowException
    This exception is thrown if this is not called in a join step.
UnhandledInMergeArtifactsException
    This exception is thrown in case of unresolved conflicts.
MissingInMergeArtifactsException
    This exception is thrown in case an artifact specified in `include` cannot
    be found.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -304,7 +304,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 6, @@ -334,10 +334,10 @@ "id": "bd1c2814", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:59.163926Z", - "iopub.status.busy": "2024-07-25T06:16:59.163856Z", - "iopub.status.idle": "2024-07-25T06:16:59.169464Z", - "shell.execute_reply": "2024-07-25T06:16:59.169261Z" + "iopub.execute_input": "2024-11-05T04:49:40.335039Z", + "iopub.status.busy": "2024-11-05T04:49:40.334903Z", + "iopub.status.idle": "2024-11-05T04:49:40.341889Z", + "shell.execute_reply": "2024-11-05T04:49:40.341510Z" }, "scrolled": true }, @@ -364,7 +364,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 7, @@ -382,10 +382,10 @@ "id": "46b05162", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:59.171289Z", - "iopub.status.busy": "2024-07-25T06:16:59.171208Z", - "iopub.status.idle": "2024-07-25T06:16:59.172794Z", - "shell.execute_reply": "2024-07-25T06:16:59.172555Z" + "iopub.execute_input": "2024-11-05T04:49:40.344369Z", + "iopub.status.busy": "2024-11-05T04:49:40.344228Z", + "iopub.status.idle": "2024-11-05T04:49:40.346236Z", + "shell.execute_reply": "2024-11-05T04:49:40.345912Z" } }, "outputs": [], @@ -435,10 +435,10 @@ "id": "0b2d482c", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:59.174707Z", - "iopub.status.busy": "2024-07-25T06:16:59.174616Z", - "iopub.status.idle": "2024-07-25T06:16:59.176984Z", - "shell.execute_reply": "2024-07-25T06:16:59.176727Z" + "iopub.execute_input": "2024-11-05T04:49:40.349463Z", + "iopub.status.busy": "2024-11-05T04:49:40.349307Z", + "iopub.status.idle": "2024-11-05T04:49:40.352041Z", + "shell.execute_reply": "2024-11-05T04:49:40.351672Z" } }, "outputs": [ @@ -461,7 +461,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 9, @@ -489,10 +489,10 @@ "id": "3b674cdb", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:59.179209Z", - "iopub.status.busy": "2024-07-25T06:16:59.179111Z", - "iopub.status.idle": "2024-07-25T06:16:59.184913Z", - "shell.execute_reply": "2024-07-25T06:16:59.184699Z" + "iopub.execute_input": "2024-11-05T04:49:40.355221Z", + "iopub.status.busy": "2024-11-05T04:49:40.355082Z", + "iopub.status.idle": "2024-11-05T04:49:40.361420Z", + "shell.execute_reply": "2024-11-05T04:49:40.361130Z" } }, "outputs": [ @@ -500,9 +500,9 @@ "data": { "text/html": [ "\n", - "

class IncludeFile name, **kwargs[source]

Includes a local file as a parameter for the flow.

`IncludeFile` behaves like `Parameter` except that it reads its value from a file instead of
the command line. The user provides a path to a file on the command line. The file contents
are saved as a read-only artifact which is available in all steps of the flow.

Parameters
----------
name : str
    User-visible parameter name.
default : Union[str, Callable[ParameterContext, str]]
    Default path to a local file. A function
    implies that the parameter corresponds to a *deploy-time parameter*.
is_text : bool, default True
    Convert the file contents to a string using the provided `encoding`.
    If False, the artifact is stored in `bytes`.
encoding : str, optional, default 'utf-8'
    Use this encoding to decode the file contexts if `is_text=True`.
required : bool, default False
    Require that the user specified a value for the parameter.
    `required=True` implies that the `default` is not used.
help : str, optional
    Help text to show in `run --help`.
show_default : bool, default True
    If True, show the default value in the help text.

\n", + "

class IncludeFile name, **kwargs[source]

Includes a local file as a parameter for the flow.

`IncludeFile` behaves like `Parameter` except that it reads its value from a file instead of
the command line. The user provides a path to a file on the command line. The file contents
are saved as a read-only artifact which is available in all steps of the flow.

Parameters
----------
name : str
    User-visible parameter name.
default : Union[str, Callable[ParameterContext, str]]
    Default path to a local file. A function
    implies that the parameter corresponds to a *deploy-time parameter*.
is_text : bool, default True
    Convert the file contents to a string using the provided `encoding`.
    If False, the artifact is stored in `bytes`.
encoding : str, optional, default 'utf-8'
    Use this encoding to decode the file contexts if `is_text=True`.
required : bool, default False
    Require that the user specified a value for the parameter.
    `required=True` implies that the `default` is not used.
help : str, optional
    Help text to show in `run --help`.
show_default : bool, default True
    If True, show the default value in the help text.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -519,7 +519,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 10, @@ -556,7 +556,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0" + "version": "3.10.13" } }, "nbformat": 4, diff --git a/docs/api/flowspec.md b/docs/api/flowspec.md index 84873f32..717a6aef 100644 --- a/docs/api/flowspec.md +++ b/docs/api/flowspec.md @@ -33,7 +33,7 @@ To query and manipulate the currently executing run inside your flow, see the [` Annotate methods that are a part of your Metaflow workflow with [the `@step` decorator](/api/step-decorators/step). Use `FlowSpec.next` to define transitions between steps: - + @@ -72,7 +72,7 @@ Use the operations below, `FlowSpec.input`, `FlowSpec.index`, and `FlowSpec.fore - + @@ -84,7 +84,7 @@ Use the operations below, `FlowSpec.input`, `FlowSpec.index`, and `FlowSpec.fore - + @@ -160,7 +160,7 @@ The function called gets a parameter `context` that contains attributes about th The `IncludeFile` object is a special `Parameter` that reads its value from a local file. For an example, see [Data in Local Files](/scaling/data#data-in-local-files). - + diff --git a/docs/api/runner.ipynb b/docs/api/runner.ipynb index c2e7497d..b35cdf0b 100644 --- a/docs/api/runner.ipynb +++ b/docs/api/runner.ipynb @@ -22,10 +22,10 @@ "id": "d4c02781", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:56.846165Z", - "iopub.status.busy": "2024-07-25T06:16:56.846050Z", - "iopub.status.idle": "2024-07-25T06:16:57.124578Z", - "shell.execute_reply": "2024-07-25T06:16:57.124277Z" + "iopub.execute_input": "2024-11-05T04:49:38.524408Z", + "iopub.status.busy": "2024-11-05T04:49:38.524273Z", + "iopub.status.idle": "2024-11-05T04:49:38.831533Z", + "shell.execute_reply": "2024-11-05T04:49:38.831068Z" } }, "outputs": [], @@ -70,10 +70,10 @@ "id": "7fdb7184", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.127133Z", - "iopub.status.busy": "2024-07-25T06:16:57.126981Z", - "iopub.status.idle": "2024-07-25T06:16:57.135119Z", - "shell.execute_reply": "2024-07-25T06:16:57.134789Z" + "iopub.execute_input": "2024-11-05T04:49:38.835936Z", + "iopub.status.busy": "2024-11-05T04:49:38.835654Z", + "iopub.status.idle": "2024-11-05T04:49:38.845893Z", + "shell.execute_reply": "2024-11-05T04:49:38.844822Z" } }, "outputs": [ @@ -81,9 +81,9 @@ "data": { "text/html": [ "\n", - "

class Runner flow_file, show_output=True, profile=None, env=None, cwd=None, **kwargs[source]

Metaflow's Runner API that presents a programmatic interface
to run flows and perform other operations either synchronously or asynchronously.
The class expects a path to the flow file along with optional arguments
that match top-level options on the command-line.

This class works as a context manager, calling `cleanup()` to remove
temporary files at exit.

Example:
```python
with Runner('slowflow.py', pylint=False) as runner:
    result = runner.run(alpha=5, tags=[\"abc\", \"def\"], max_workers=5)
    print(result.run.finished)
```

Parameters
----------
flow_file : str
    Path to the flow file to run
show_output : bool, default True
    Show the 'stdout' and 'stderr' to the console by default,
    Only applicable for synchronous 'run' and 'resume' functions.
profile : Optional[str], default None
    Metaflow profile to use to run this run. If not specified, the default
    profile is used (or the one already set using `METAFLOW_PROFILE`)
env : Optional[Dict], default None
    Additional environment variables to set for the Run. This overrides the
    environment set for this process.
cwd : Optional[str], default None
    The directory to run the subprocess in; if not specified, the current
    directory is used.
**kwargs : Any
    Additional arguments that you would pass to `python myflow.py` before
    the `run` command.

\n", + "

class Runner flow_file, show_output=True, profile=None, env=None, cwd=None, **kwargs[source]

Metaflow's Runner API that presents a programmatic interface
to run flows and perform other operations either synchronously or asynchronously.
The class expects a path to the flow file along with optional arguments
that match top-level options on the command-line.

This class works as a context manager, calling `cleanup()` to remove
temporary files at exit.

Example:
```python
with Runner('slowflow.py', pylint=False) as runner:
    result = runner.run(alpha=5, tags=[\"abc\", \"def\"], max_workers=5)
    print(result.run.finished)
```

Parameters
----------
flow_file : str
    Path to the flow file to run
show_output : bool, default True
    Show the 'stdout' and 'stderr' to the console by default,
    Only applicable for synchronous 'run' and 'resume' functions.
profile : str, optional, default None
    Metaflow profile to use to run this run. If not specified, the default
    profile is used (or the one already set using `METAFLOW_PROFILE`)
env : Dict[str, str], optional, default None
    Additional environment variables to set for the Run. This overrides the
    environment set for this process.
cwd : str, optional, default None
    The directory to run the subprocess in; if not specified, the current
    directory is used.
file_read_timeout : int, default 3600
    The timeout until which we try to read the runner attribute file.
**kwargs : Any
    Additional arguments that you would pass to `python myflow.py` before
    the `run` command.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -91,15 +91,16 @@ "\n", "\t\n", "\t\n", - "\t\n", - "\t\n", - "\t\n", + "\t\n", + "\t\n", + "\t\n", + "\t\n", "\t\n", "\n", "" ], "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -117,10 +118,10 @@ "id": "84c83820", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.137507Z", - "iopub.status.busy": "2024-07-25T06:16:57.137393Z", - "iopub.status.idle": "2024-07-25T06:16:57.139946Z", - "shell.execute_reply": "2024-07-25T06:16:57.139696Z" + "iopub.execute_input": "2024-11-05T04:49:38.849518Z", + "iopub.status.busy": "2024-11-05T04:49:38.849315Z", + "iopub.status.idle": "2024-11-05T04:49:38.853575Z", + "shell.execute_reply": "2024-11-05T04:49:38.852546Z" } }, "outputs": [ @@ -128,9 +129,9 @@ "data": { "text/html": [ "\n", - "

method Runner.cleanup (self)[source]

Delete any temporary files created during execution.

\n", + "

method Runner.cleanup (self)[source]

Delete any temporary files created during execution.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -139,7 +140,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 3, @@ -167,10 +168,10 @@ "id": "8c61365e", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.142811Z", - "iopub.status.busy": "2024-07-25T06:16:57.142681Z", - "iopub.status.idle": "2024-07-25T06:16:57.145665Z", - "shell.execute_reply": "2024-07-25T06:16:57.145412Z" + "iopub.execute_input": "2024-11-05T04:49:38.857487Z", + "iopub.status.busy": "2024-11-05T04:49:38.857309Z", + "iopub.status.idle": "2024-11-05T04:49:38.861539Z", + "shell.execute_reply": "2024-11-05T04:49:38.860972Z" } }, "outputs": [ @@ -178,9 +179,9 @@ "data": { "text/html": [ "\n", - "

method Runner.run (self, **kwargs) -> metaflow.runner.metaflow_runner.ExecutingRun[source]

Blocking execution of the run. This method will wait until
the run has completed execution.

Parameters
----------
**kwargs : Any
    Additional arguments that you would pass to `python myflow.py` after
    the `run` command, in particular, any parameters accepted by the flow.

Returns
-------
ExecutingRun
    ExecutingRun containing the results of the run.

\n", + "

method Runner.run (self, **kwargs) -> metaflow.runner.metaflow_runner.ExecutingRun[source]

Blocking execution of the run. This method will wait until
the run has completed execution.

Parameters
----------
**kwargs : Any
    Additional arguments that you would pass to `python myflow.py` after
    the `run` command, in particular, any parameters accepted by the flow.

Returns
-------
ExecutingRun
    ExecutingRun containing the results of the run.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -194,7 +195,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 4, @@ -212,10 +213,10 @@ "id": "4fe8757c", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.147705Z", - "iopub.status.busy": "2024-07-25T06:16:57.147607Z", - "iopub.status.idle": "2024-07-25T06:16:57.150376Z", - "shell.execute_reply": "2024-07-25T06:16:57.150071Z" + "iopub.execute_input": "2024-11-05T04:49:38.865336Z", + "iopub.status.busy": "2024-11-05T04:49:38.865137Z", + "iopub.status.idle": "2024-11-05T04:49:38.869113Z", + "shell.execute_reply": "2024-11-05T04:49:38.868447Z" } }, "outputs": [ @@ -223,9 +224,9 @@ "data": { "text/html": [ "\n", - "

method Runner.resume (self, **kwargs)[source]

Blocking resume execution of the run.
This method will wait until the resumed run has completed execution.

Parameters
----------
**kwargs : Any
    Additional arguments that you would pass to `python ./myflow.py` after
    the `resume` command.

Returns
-------
ExecutingRun
    ExecutingRun containing the results of the resumed run.

\n", + "

method Runner.resume (self, **kwargs)[source]

Blocking resume execution of the run.
This method will wait until the resumed run has completed execution.

Parameters
----------
**kwargs : Any
    Additional arguments that you would pass to `python ./myflow.py` after
    the `resume` command.

Returns
-------
ExecutingRun
    ExecutingRun containing the results of the resumed run.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -239,7 +240,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 5, @@ -265,10 +266,10 @@ "id": "ec91c1c9", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.152518Z", - "iopub.status.busy": "2024-07-25T06:16:57.152422Z", - "iopub.status.idle": "2024-07-25T06:16:57.155118Z", - "shell.execute_reply": "2024-07-25T06:16:57.154859Z" + "iopub.execute_input": "2024-11-05T04:49:38.873881Z", + "iopub.status.busy": "2024-11-05T04:49:38.873349Z", + "iopub.status.idle": "2024-11-05T04:49:38.877474Z", + "shell.execute_reply": "2024-11-05T04:49:38.876752Z" } }, "outputs": [ @@ -276,9 +277,9 @@ "data": { "text/html": [ "\n", - "

method Runner.async_run (self, **kwargs) -> metaflow.runner.metaflow_runner.ExecutingRun[source]

Non-blocking execution of the run. This method will return as soon as the
run has launched.

Note that this method is asynchronous and needs to be `await`ed.

Parameters
----------
**kwargs : Any
    Additional arguments that you would pass to `python myflow.py` after
    the `run` command, in particular, any parameters accepted by the flow.

Returns
-------
ExecutingRun
    ExecutingRun representing the run that was started.

\n", + "

method Runner.async_run (self, **kwargs) -> metaflow.runner.metaflow_runner.ExecutingRun[source]

Non-blocking execution of the run. This method will return as soon as the
run has launched.

Note that this method is asynchronous and needs to be `await`ed.

Parameters
----------
**kwargs : Any
    Additional arguments that you would pass to `python myflow.py` after
    the `run` command, in particular, any parameters accepted by the flow.

Returns
-------
ExecutingRun
    ExecutingRun representing the run that was started.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -292,7 +293,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 6, @@ -310,10 +311,10 @@ "id": "87f8530a", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.157634Z", - "iopub.status.busy": "2024-07-25T06:16:57.157548Z", - "iopub.status.idle": "2024-07-25T06:16:57.160071Z", - "shell.execute_reply": "2024-07-25T06:16:57.159813Z" + "iopub.execute_input": "2024-11-05T04:49:38.880926Z", + "iopub.status.busy": "2024-11-05T04:49:38.880626Z", + "iopub.status.idle": "2024-11-05T04:49:38.884798Z", + "shell.execute_reply": "2024-11-05T04:49:38.884096Z" } }, "outputs": [ @@ -321,9 +322,9 @@ "data": { "text/html": [ "\n", - "

method Runner.async_resume (self, **kwargs)[source]

Non-blocking resume execution of the run.
This method will return as soon as the resume has launched.

Note that this method is asynchronous and needs to be `await`ed.

Parameters
----------
**kwargs : Any
    Additional arguments that you would pass to `python myflow.py` after
    the `resume` command.

Returns
-------
ExecutingRun
    ExecutingRun representing the resumed run that was started.

\n", + "

method Runner.async_resume (self, **kwargs)[source]

Non-blocking resume execution of the run.
This method will return as soon as the resume has launched.

Note that this method is asynchronous and needs to be `await`ed.

Parameters
----------
**kwargs : Any
    Additional arguments that you would pass to `python myflow.py` after
    the `resume` command.

Returns
-------
ExecutingRun
    ExecutingRun representing the resumed run that was started.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -337,7 +338,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 7, @@ -365,10 +366,10 @@ "id": "9fb445c1", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.162889Z", - "iopub.status.busy": "2024-07-25T06:16:57.162793Z", - "iopub.status.idle": "2024-07-25T06:16:57.167115Z", - "shell.execute_reply": "2024-07-25T06:16:57.166894Z" + "iopub.execute_input": "2024-11-05T04:49:38.888857Z", + "iopub.status.busy": "2024-11-05T04:49:38.888665Z", + "iopub.status.idle": "2024-11-05T04:49:38.894045Z", + "shell.execute_reply": "2024-11-05T04:49:38.893671Z" } }, "outputs": [ @@ -376,9 +377,9 @@ "data": { "text/html": [ "\n", - "

class NBRunner flow, show_output=True, profile=None, env=None, base_dir=None, **kwargs[source]

A  wrapper over `Runner` for executing flows defined in a Jupyter
notebook cell.

Instantiate this class on the last line of a notebook cell where
a `flow` is defined. In contrast to `Runner`, this class is not
meant to be used in a context manager. Instead, use a blocking helper
function like `nbrun` (which calls `cleanup()` internally) or call
`cleanup()` explictly when using non-blocking APIs.

```python
run = NBRunner(FlowName).nbrun()
```

Parameters
----------
flow : FlowSpec
    Flow defined in the same cell
show_output : bool, default True
    Show the 'stdout' and 'stderr' to the console by default,
    Only applicable for synchronous 'run' and 'resume' functions.
profile : Optional[str], default None
    Metaflow profile to use to run this run. If not specified, the default
    profile is used (or the one already set using `METAFLOW_PROFILE`)
env : Optional[Dict], default None
    Additional environment variables to set for the Run. This overrides the
    environment set for this process.
base_dir : Optional[str], default None
    The directory to run the subprocess in; if not specified, a temporary
    directory is used.
**kwargs : Any
    Additional arguments that you would pass to `python myflow.py` before
    the `run` command.

\n", + "

class NBRunner flow, show_output=True, profile=None, env=None, base_dir=None, **kwargs[source]

A  wrapper over `Runner` for executing flows defined in a Jupyter
notebook cell.

Instantiate this class on the last line of a notebook cell where
a `flow` is defined. In contrast to `Runner`, this class is not
meant to be used in a context manager. Instead, use a blocking helper
function like `nbrun` (which calls `cleanup()` internally) or call
`cleanup()` explictly when using non-blocking APIs.

```python
run = NBRunner(FlowName).nbrun()
```

Parameters
----------
flow : FlowSpec
    Flow defined in the same cell
show_output : bool, default True
    Show the 'stdout' and 'stderr' to the console by default,
    Only applicable for synchronous 'run' and 'resume' functions.
profile : str, optional, default None
    Metaflow profile to use to run this run. If not specified, the default
    profile is used (or the one already set using `METAFLOW_PROFILE`)
env : Dict[str, str], optional, default None
    Additional environment variables to set for the Run. This overrides the
    environment set for this process.
base_dir : str, optional, default None
    The directory to run the subprocess in; if not specified, the current
    working directory is used.
file_read_timeout : int, default 3600
    The timeout until which we try to read the runner attribute file.
**kwargs : Any
    Additional arguments that you would pass to `python myflow.py` before
    the `run` command.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -386,15 +387,16 @@ "\n", "\t\n", "\t\n", - "\t\n", - "\t\n", - "\t\n", + "\t\n", + "\t\n", + "\t\n", + "\t\n", "\t\n", "\n", "" ], "text/plain": [ - "" + "" ] }, "execution_count": 8, @@ -420,10 +422,10 @@ "id": "8971ea07", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.168964Z", - "iopub.status.busy": "2024-07-25T06:16:57.168888Z", - "iopub.status.idle": "2024-07-25T06:16:57.171468Z", - "shell.execute_reply": "2024-07-25T06:16:57.171219Z" + "iopub.execute_input": "2024-11-05T04:49:38.897662Z", + "iopub.status.busy": "2024-11-05T04:49:38.897446Z", + "iopub.status.idle": "2024-11-05T04:49:38.900390Z", + "shell.execute_reply": "2024-11-05T04:49:38.900083Z" } }, "outputs": [ @@ -431,9 +433,9 @@ "data": { "text/html": [ "\n", - "

method NBRunner.nbrun (self, **kwargs)[source]

Blocking execution of the run. This method will wait until
the run has completed execution.

Note that in contrast to `run`, this method returns a
`metaflow.Run` object directly and calls `cleanup()` internally
to support a common notebook pattern of executing a flow and
retrieving its results immediately.

Parameters
----------
**kwargs : Any
    Additional arguments that you would pass to `python myflow.py` after
    the `run` command, in particular, any parameters accepted by the flow.

Returns
-------
Run
    A `metaflow.Run` object representing the finished run.

\n", + "

method NBRunner.nbrun (self, **kwargs)[source]

Blocking execution of the run. This method will wait until
the run has completed execution.

Note that in contrast to `run`, this method returns a
`metaflow.Run` object directly and calls `cleanup()` internally
to support a common notebook pattern of executing a flow and
retrieving its results immediately.

Parameters
----------
**kwargs : Any
    Additional arguments that you would pass to `python myflow.py` after
    the `run` command, in particular, any parameters accepted by the flow.

Returns
-------
Run
    A `metaflow.Run` object representing the finished run.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -447,7 +449,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 9, @@ -465,10 +467,10 @@ "id": "95a8c5b3", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.173679Z", - "iopub.status.busy": "2024-07-25T06:16:57.173586Z", - "iopub.status.idle": "2024-07-25T06:16:57.176168Z", - "shell.execute_reply": "2024-07-25T06:16:57.175862Z" + "iopub.execute_input": "2024-11-05T04:49:38.903312Z", + "iopub.status.busy": "2024-11-05T04:49:38.903104Z", + "iopub.status.idle": "2024-11-05T04:49:38.906095Z", + "shell.execute_reply": "2024-11-05T04:49:38.905687Z" } }, "outputs": [ @@ -476,9 +478,9 @@ "data": { "text/html": [ "\n", - "

method NBRunner.nbresume (self, **kwargs)[source]

Blocking resuming of a run. This method will wait until
the resumed run has completed execution.

Note that in contrast to `resume`, this method returns a
`metaflow.Run` object directly and calls `cleanup()` internally
to support a common notebook pattern of executing a flow and
retrieving its results immediately.

Parameters
----------
**kwargs : Any
    Additional arguments that you would pass to `python myflow.py` after
    the `resume` command.

Returns
-------
Run
    A `metaflow.Run` object representing the resumed run.

\n", + "

method NBRunner.nbresume (self, **kwargs)[source]

Blocking resuming of a run. This method will wait until
the resumed run has completed execution.

Note that in contrast to `resume`, this method returns a
`metaflow.Run` object directly and calls `cleanup()` internally
to support a common notebook pattern of executing a flow and
retrieving its results immediately.

Parameters
----------
**kwargs : Any
    Additional arguments that you would pass to `python myflow.py` after
    the `resume` command.

Returns
-------
Run
    A `metaflow.Run` object representing the resumed run.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -492,7 +494,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 10, @@ -518,10 +520,10 @@ "id": "e9360c97", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.178155Z", - "iopub.status.busy": "2024-07-25T06:16:57.178052Z", - "iopub.status.idle": "2024-07-25T06:16:57.180530Z", - "shell.execute_reply": "2024-07-25T06:16:57.180242Z" + "iopub.execute_input": "2024-11-05T04:49:38.909278Z", + "iopub.status.busy": "2024-11-05T04:49:38.909128Z", + "iopub.status.idle": "2024-11-05T04:49:38.911814Z", + "shell.execute_reply": "2024-11-05T04:49:38.911513Z" } }, "outputs": [ @@ -529,9 +531,9 @@ "data": { "text/html": [ "\n", - "

method NBRunner.async_run (self, **kwargs)[source]

Non-blocking execution of the run. This method will return as soon as the
run has launched. This method is equivalent to `Runner.async_run`.

Note that this method is asynchronous and needs to be `await`ed.


Parameters
----------
**kwargs : Any
    Additional arguments that you would pass to `python myflow.py` after
    the `run` command, in particular, any parameters accepted by the flow.

Returns
-------
ExecutingRun
    ExecutingRun representing the run that was started.

\n", + "

method NBRunner.async_run (self, **kwargs)[source]

Non-blocking execution of the run. This method will return as soon as the
run has launched. This method is equivalent to `Runner.async_run`.

Note that this method is asynchronous and needs to be `await`ed.


Parameters
----------
**kwargs : Any
    Additional arguments that you would pass to `python myflow.py` after
    the `run` command, in particular, any parameters accepted by the flow.

Returns
-------
ExecutingRun
    ExecutingRun representing the run that was started.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -545,7 +547,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 11, @@ -563,10 +565,10 @@ "id": "dd48a525", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.182821Z", - "iopub.status.busy": "2024-07-25T06:16:57.182745Z", - "iopub.status.idle": "2024-07-25T06:16:57.185040Z", - "shell.execute_reply": "2024-07-25T06:16:57.184827Z" + "iopub.execute_input": "2024-11-05T04:49:38.915378Z", + "iopub.status.busy": "2024-11-05T04:49:38.915233Z", + "iopub.status.idle": "2024-11-05T04:49:38.918145Z", + "shell.execute_reply": "2024-11-05T04:49:38.917863Z" } }, "outputs": [ @@ -574,9 +576,9 @@ "data": { "text/html": [ "\n", - "

method NBRunner.async_resume (self, **kwargs)[source]

Non-blocking execution of the run. This method will return as soon as the
run has launched. This method is equivalent to `Runner.async_resume`.

Note that this method is asynchronous and needs to be `await`ed.

Parameters
----------
**kwargs : Any
    Additional arguments that you would pass to `python myflow.py` after
    the `run` command, in particular, any parameters accepted by the flow.

Returns
-------
ExecutingRun
    ExecutingRun representing the run that was started.

\n", + "

method NBRunner.async_resume (self, **kwargs)[source]

Non-blocking execution of the run. This method will return as soon as the
run has launched. This method is equivalent to `Runner.async_resume`.

Note that this method is asynchronous and needs to be `await`ed.

Parameters
----------
**kwargs : Any
    Additional arguments that you would pass to `python myflow.py` after
    the `run` command, in particular, any parameters accepted by the flow.

Returns
-------
ExecutingRun
    ExecutingRun representing the run that was started.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -590,7 +592,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 12, @@ -608,10 +610,10 @@ "id": "4dcbcf6a", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.187052Z", - "iopub.status.busy": "2024-07-25T06:16:57.186966Z", - "iopub.status.idle": "2024-07-25T06:16:57.189166Z", - "shell.execute_reply": "2024-07-25T06:16:57.188955Z" + "iopub.execute_input": "2024-11-05T04:49:38.921219Z", + "iopub.status.busy": "2024-11-05T04:49:38.921107Z", + "iopub.status.idle": "2024-11-05T04:49:38.923916Z", + "shell.execute_reply": "2024-11-05T04:49:38.923602Z" } }, "outputs": [ @@ -619,9 +621,9 @@ "data": { "text/html": [ "\n", - "

method NBRunner.cleanup (self)[source]

Delete any temporary files created during execution.

Call this method after using `async_run` or `async_resume`. You don't
have to call this after `nbrun` or `nbresume`.

\n", + "

method NBRunner.cleanup (self)[source]

Delete any temporary files created during execution.

Call this method after using `async_run` or `async_resume`. You don't
have to call this after `nbrun` or `nbresume`.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -630,7 +632,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 13, @@ -656,10 +658,10 @@ "id": "ca407a51", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.191237Z", - "iopub.status.busy": "2024-07-25T06:16:57.191153Z", - "iopub.status.idle": "2024-07-25T06:16:57.195115Z", - "shell.execute_reply": "2024-07-25T06:16:57.194853Z" + "iopub.execute_input": "2024-11-05T04:49:38.927062Z", + "iopub.status.busy": "2024-11-05T04:49:38.926832Z", + "iopub.status.idle": "2024-11-05T04:49:38.931538Z", + "shell.execute_reply": "2024-11-05T04:49:38.931226Z" } }, "outputs": [ @@ -667,9 +669,9 @@ "data": { "text/html": [ "\n", - "

class ExecutingRun [source]

This class contains a reference to a `metaflow.Run` object representing
the currently executing or finished run, as well as metadata related
to the process.

`ExecutingRun` is returned by methods in `Runner` and `NBRunner`. It is not
meant to be instantiated directly.

This class works as a context manager, allowing you to use a pattern like
```python
with Runner(...).run() as running:
    ...
```
Note that you should use either this object as the context manager or
`Runner`, not both in a nested manner.

\n", + "

class ExecutingRun [source]

This class contains a reference to a `metaflow.Run` object representing
the currently executing or finished run, as well as metadata related
to the process.

`ExecutingRun` is returned by methods in `Runner` and `NBRunner`. It is not
meant to be instantiated directly.

This class works as a context manager, allowing you to use a pattern like
```python
with Runner(...).run() as running:
    ...
```
Note that you should use either this object as the context manager or
`Runner`, not both in a nested manner.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -678,7 +680,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 14, @@ -696,10 +698,10 @@ "id": "f127a019", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.197432Z", - "iopub.status.busy": "2024-07-25T06:16:57.197357Z", - "iopub.status.idle": "2024-07-25T06:16:57.199514Z", - "shell.execute_reply": "2024-07-25T06:16:57.199252Z" + "iopub.execute_input": "2024-11-05T04:49:38.934714Z", + "iopub.status.busy": "2024-11-05T04:49:38.934541Z", + "iopub.status.idle": "2024-11-05T04:49:38.936991Z", + "shell.execute_reply": "2024-11-05T04:49:38.936694Z" } }, "outputs": [ @@ -718,7 +720,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 15, @@ -736,10 +738,10 @@ "id": "777973d3", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.201633Z", - "iopub.status.busy": "2024-07-25T06:16:57.201538Z", - "iopub.status.idle": "2024-07-25T06:16:57.203616Z", - "shell.execute_reply": "2024-07-25T06:16:57.203359Z" + "iopub.execute_input": "2024-11-05T04:49:38.940497Z", + "iopub.status.busy": "2024-11-05T04:49:38.940268Z", + "iopub.status.idle": "2024-11-05T04:49:38.942816Z", + "shell.execute_reply": "2024-11-05T04:49:38.942508Z" } }, "outputs": [ @@ -747,18 +749,18 @@ "data": { "text/html": [ "\n", - "

property ExecutingRun.status [source]

Returns the status of the underlying subprocess that is responsible
for executing the run.

The return value is one of the following strings:
- `running` indicates a currently executing run.
- `failed` indicates a failed run.
- `successful` a successful run.

Returns
-------
str
    The current status of the run.

\n", + "

property ExecutingRun.status [source]

Returns the status of the underlying subprocess that is responsible
for executing the run.

The return value is one of the following strings:
- `timeout` indicates that the run timed out.
- `running` indicates a currently executing run.
- `failed` indicates a failed run.
- `successful` indicates a successful run.

Returns
-------
str
    The current status of the run.

\n", "\n", "\n", "\n", - "\n", + "\n", "\n", "\n", "\n", "" ], "text/plain": [ - "" + "" ] }, "execution_count": 16, @@ -776,10 +778,10 @@ "id": "ca5b461d", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.205776Z", - "iopub.status.busy": "2024-07-25T06:16:57.205682Z", - "iopub.status.idle": "2024-07-25T06:16:57.207721Z", - "shell.execute_reply": "2024-07-25T06:16:57.207475Z" + "iopub.execute_input": "2024-11-05T04:49:38.946402Z", + "iopub.status.busy": "2024-11-05T04:49:38.946253Z", + "iopub.status.idle": "2024-11-05T04:49:38.948849Z", + "shell.execute_reply": "2024-11-05T04:49:38.948519Z" } }, "outputs": [ @@ -798,7 +800,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 17, @@ -816,10 +818,10 @@ "id": "c6b98164", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.209801Z", - "iopub.status.busy": "2024-07-25T06:16:57.209713Z", - "iopub.status.idle": "2024-07-25T06:16:57.211588Z", - "shell.execute_reply": "2024-07-25T06:16:57.211359Z" + "iopub.execute_input": "2024-11-05T04:49:38.952311Z", + "iopub.status.busy": "2024-11-05T04:49:38.952172Z", + "iopub.status.idle": "2024-11-05T04:49:38.954815Z", + "shell.execute_reply": "2024-11-05T04:49:38.954384Z" } }, "outputs": [ @@ -838,7 +840,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 18, @@ -864,10 +866,10 @@ "id": "f716f347", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.214150Z", - "iopub.status.busy": "2024-07-25T06:16:57.214062Z", - "iopub.status.idle": "2024-07-25T06:16:57.216506Z", - "shell.execute_reply": "2024-07-25T06:16:57.216287Z" + "iopub.execute_input": "2024-11-05T04:49:38.958396Z", + "iopub.status.busy": "2024-11-05T04:49:38.958277Z", + "iopub.status.idle": "2024-11-05T04:49:38.961278Z", + "shell.execute_reply": "2024-11-05T04:49:38.960927Z" }, "scrolled": true }, @@ -876,16 +878,16 @@ "data": { "text/html": [ "\n", - "

method ExecutingRun.wait (self, timeout: Optional[float] = None, stream: Optional[str] = None) -> 'ExecutingRun'[source]

Wait for this run to finish, optionally with a timeout
and optionally streaming its output.

Note that this method is asynchronous and needs to be `await`ed.

Parameters
----------
timeout : Optional[float], default None
    The maximum time to wait for the run to finish.
    If the timeout is reached, the run is terminated
stream : Optional[str], default None
    If specified, the specified stream is printed to stdout. `stream` can
    be one of `stdout` or `stderr`.

Returns
-------
ExecutingRun
    This object, allowing you to chain calls.

\n", + "

method ExecutingRun.wait (self, timeout: Optional[float] = None, stream: Optional[str] = None) -> 'ExecutingRun'[source]

Wait for this run to finish, optionally with a timeout
and optionally streaming its output.

Note that this method is asynchronous and needs to be `await`ed.

Parameters
----------
timeout : float, optional, default None
    The maximum time, in seconds, to wait for the run to finish.
    If the timeout is reached, the run is terminated. If not specified, wait
    forever.
stream : str, optional, default None
    If specified, the specified stream is printed to stdout. `stream` can
    be one of `stdout` or `stderr`.

Returns
-------
ExecutingRun
    This object, allowing you to chain calls.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", "\n", "\n", - "\t\n", - "\t\n", + "\t\n", + "\t\n", "\n", "\n", "\t\n", @@ -893,7 +895,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 19, @@ -911,10 +913,10 @@ "id": "4b97379d", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:57.218736Z", - "iopub.status.busy": "2024-07-25T06:16:57.218656Z", - "iopub.status.idle": "2024-07-25T06:16:57.220972Z", - "shell.execute_reply": "2024-07-25T06:16:57.220747Z" + "iopub.execute_input": "2024-11-05T04:49:38.964680Z", + "iopub.status.busy": "2024-11-05T04:49:38.964522Z", + "iopub.status.idle": "2024-11-05T04:49:38.968153Z", + "shell.execute_reply": "2024-11-05T04:49:38.967677Z" } }, "outputs": [ @@ -922,16 +924,16 @@ "data": { "text/html": [ "\n", - "

method ExecutingRun.stream_log (self, stream: str, position: Optional[int] = None) -> Iterator[Tuple[int, str]][source]

Asynchronous iterator to stream logs from the subprocess line by line.

Note that this method is asynchronous and needs to be `await`ed.

Parameters
----------
stream : str
    The stream to stream logs from. Can be one of `stdout` or `stderr`.
position : Optional[int], default None
    The position in the log file to start streaming from. If None, it starts
    from the beginning of the log file. This allows resuming streaming from
    a previously known position

Yields
------
Tuple[int, str]
    A tuple containing the position in the log file and the line read. The
    position returned can be used to feed into another `stream_logs` call
    for example.

\n", + "

method ExecutingRun.stream_log (self, stream: str, position: Optional[int] = None) -> Iterator[Tuple[int, str]][source]

Asynchronous iterator to stream logs from the subprocess line by line.

Note that this method is asynchronous and needs to be `await`ed.

Parameters
----------
stream : str
    The stream to stream logs from. Can be one of `stdout` or `stderr`.
position : int, optional, default None
    The position in the log file to start streaming from. If None, it starts
    from the beginning of the log file. This allows resuming streaming from
    a previously known position

Yields
------
Tuple[int, str]
    A tuple containing the position in the log file and the line read. The
    position returned can be used to feed into another `stream_logs` call
    for example.

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", "\n", "\n", "\t\n", - "\t\n", + "\t\n", "\n", "\n", "\t\n", @@ -939,7 +941,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 20, @@ -976,7 +978,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0" + "version": "3.10.13" } }, "nbformat": 4, diff --git a/docs/api/runner.md b/docs/api/runner.md index 4b148f48..f8cd70a4 100644 --- a/docs/api/runner.md +++ b/docs/api/runner.md @@ -24,7 +24,7 @@ with await Runner(...).async_run() as running: If you don't use `Runner` as a context manager, remember to call `Runner.cleanup()` to remove any leftover temp files. - + @@ -32,16 +32,17 @@ If you don't use `Runner` as a context manager, remember to call `Runner.cleanup - - - + + + + - + @@ -55,7 +56,7 @@ If you don't use `Runner` as a context manager, remember to call `Runner.cleanup These calls block until the command completes. - + @@ -70,7 +71,7 @@ These calls block until the command completes. - + @@ -87,7 +88,7 @@ These calls block until the command completes. ### Non-Blocking API - + @@ -102,7 +103,7 @@ These calls block until the command completes. - + @@ -121,7 +122,7 @@ These calls block until the command completes. `NBRunner` is a wrapper over `Runner` which allows you to refer to a flow defined in a notebook cell instead of a file. For examples, see [Running flows in a notebook](/metaflow/managing-flows/notebook-runs). - + @@ -129,9 +130,10 @@ These calls block until the command completes. - - - + + + + @@ -140,7 +142,7 @@ These calls block until the command completes. ### Blocking API - + @@ -155,7 +157,7 @@ These calls block until the command completes. - + @@ -172,7 +174,7 @@ These calls block until the command completes. ### Non-Blocking API - + @@ -187,7 +189,7 @@ These calls block until the command completes. - + @@ -202,7 +204,7 @@ These calls block until the command completes. - + @@ -214,7 +216,7 @@ These calls block until the command completes. ## ExecutingRun - + @@ -236,7 +238,7 @@ These calls block until the command completes. - + @@ -266,14 +268,14 @@ These calls block until the command completes. ### Non-Blocking API - + - - + + @@ -282,16 +284,17 @@ These calls block until the command completes. - + - + + diff --git a/docs/api/step-decorators/batch.ipynb b/docs/api/step-decorators/batch.ipynb index 050f84df..5f09ae85 100644 --- a/docs/api/step-decorators/batch.ipynb +++ b/docs/api/step-decorators/batch.ipynb @@ -24,10 +24,10 @@ "id": "a5ef9454", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:59.400491Z", - "iopub.status.busy": "2024-07-25T06:16:59.400293Z", - "iopub.status.idle": "2024-07-25T06:16:59.685244Z", - "shell.execute_reply": "2024-07-25T06:16:59.684870Z" + "iopub.execute_input": "2024-11-05T04:49:40.485114Z", + "iopub.status.busy": "2024-11-05T04:49:40.484953Z", + "iopub.status.idle": "2024-11-05T04:49:40.776181Z", + "shell.execute_reply": "2024-11-05T04:49:40.775830Z" } }, "outputs": [], @@ -46,10 +46,10 @@ "id": "09970e68", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:59.687754Z", - "iopub.status.busy": "2024-07-25T06:16:59.687597Z", - "iopub.status.idle": "2024-07-25T06:16:59.696794Z", - "shell.execute_reply": "2024-07-25T06:16:59.696531Z" + "iopub.execute_input": "2024-11-05T04:49:40.779461Z", + "iopub.status.busy": "2024-11-05T04:49:40.779165Z", + "iopub.status.idle": "2024-11-05T04:49:40.789651Z", + "shell.execute_reply": "2024-11-05T04:49:40.789388Z" } }, "outputs": [ @@ -89,7 +89,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -126,7 +126,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0" + "version": "3.10.13" } }, "nbformat": 4, diff --git a/docs/api/step-decorators/card.ipynb b/docs/api/step-decorators/card.ipynb index 1b4dfe16..994464a5 100644 --- a/docs/api/step-decorators/card.ipynb +++ b/docs/api/step-decorators/card.ipynb @@ -16,10 +16,10 @@ "id": "20310c8b", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:17:02.423148Z", - "iopub.status.busy": "2024-07-25T06:17:02.422995Z", - "iopub.status.idle": "2024-07-25T06:17:02.722995Z", - "shell.execute_reply": "2024-07-25T06:17:02.722698Z" + "iopub.execute_input": "2024-11-05T04:49:43.306011Z", + "iopub.status.busy": "2024-11-05T04:49:43.305899Z", + "iopub.status.idle": "2024-11-05T04:49:43.587064Z", + "shell.execute_reply": "2024-11-05T04:49:43.586154Z" } }, "outputs": [], @@ -38,10 +38,10 @@ "id": "7f2bc67e", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:17:02.725533Z", - "iopub.status.busy": "2024-07-25T06:17:02.725351Z", - "iopub.status.idle": "2024-07-25T06:17:02.733139Z", - "shell.execute_reply": "2024-07-25T06:17:02.732917Z" + "iopub.execute_input": "2024-11-05T04:49:43.590804Z", + "iopub.status.busy": "2024-11-05T04:49:43.590581Z", + "iopub.status.idle": "2024-11-05T04:49:43.601149Z", + "shell.execute_reply": "2024-11-05T04:49:43.600544Z" } }, "outputs": [ @@ -49,13 +49,13 @@ "name": "stderr", "output_type": "stream", "text": [ - "/Users/ville/mambaforge/envs/docs/lib/python3.11/site-packages/numpydoc/docscrape.py:434: UserWarning: potentially wrong underline length... \n", + "/Users/rcledat/mambaforge/envs/metaflow-oss-docs/lib/python3.10/site-packages/numpydoc/docscrape.py:434: UserWarning: potentially wrong underline length... \n", "@@ Returns \n", "------- in \n", "Creates a human-readable report, a Metaflow Card, after this step completes.\n", - "... in the docstring of CardDecorator in /Users/ville/src/madhur-metaflow/metaflow/plugins/cards/card_decorator.py.\n", + "... in the docstring of CardDecorator in /Users/rcledat/mambaforge/envs/metaflow-oss-docs/lib/python3.10/site-packages/metaflow/plugins/cards/card_decorator.py.\n", " warn(msg)\n", - "/Users/ville/mambaforge/envs/docs/lib/python3.11/site-packages/numpydoc/docscrape.py:434: UserWarning: Unknown section Mf Add To Current in the docstring of CardDecorator in /Users/ville/src/madhur-metaflow/metaflow/plugins/cards/card_decorator.py.\n", + "/Users/rcledat/mambaforge/envs/metaflow-oss-docs/lib/python3.10/site-packages/numpydoc/docscrape.py:434: UserWarning: Unknown section Mf Add To Current in the docstring of CardDecorator in /Users/rcledat/mambaforge/envs/metaflow-oss-docs/lib/python3.10/site-packages/metaflow/plugins/cards/card_decorator.py.\n", " warn(msg)\n" ] }, @@ -79,7 +79,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -116,7 +116,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0" + "version": "3.10.13" } }, "nbformat": 4, diff --git a/docs/api/step-decorators/catch.ipynb b/docs/api/step-decorators/catch.ipynb index 5e4039ba..eb3384ce 100644 --- a/docs/api/step-decorators/catch.ipynb +++ b/docs/api/step-decorators/catch.ipynb @@ -18,10 +18,10 @@ "id": "627487a4", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:17:01.010753Z", - "iopub.status.busy": "2024-07-25T06:17:01.010654Z", - "iopub.status.idle": "2024-07-25T06:17:01.277105Z", - "shell.execute_reply": "2024-07-25T06:17:01.276760Z" + "iopub.execute_input": "2024-11-05T04:49:42.168151Z", + "iopub.status.busy": "2024-11-05T04:49:42.168052Z", + "iopub.status.idle": "2024-11-05T04:49:42.448742Z", + "shell.execute_reply": "2024-11-05T04:49:42.448433Z" } }, "outputs": [], @@ -40,10 +40,10 @@ "id": "af50a6d2", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:17:01.279312Z", - "iopub.status.busy": "2024-07-25T06:17:01.279168Z", - "iopub.status.idle": "2024-07-25T06:17:01.284680Z", - "shell.execute_reply": "2024-07-25T06:17:01.284461Z" + "iopub.execute_input": "2024-11-05T04:49:42.452347Z", + "iopub.status.busy": "2024-11-05T04:49:42.452164Z", + "iopub.status.idle": "2024-11-05T04:49:42.458855Z", + "shell.execute_reply": "2024-11-05T04:49:42.458494Z" } }, "outputs": [ @@ -65,7 +65,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -102,7 +102,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0" + "version": "3.10.13" } }, "nbformat": 4, diff --git a/docs/api/step-decorators/conda.ipynb b/docs/api/step-decorators/conda.ipynb index 36037f48..ddfa1394 100644 --- a/docs/api/step-decorators/conda.ipynb +++ b/docs/api/step-decorators/conda.ipynb @@ -18,10 +18,10 @@ "id": "8d5bb116", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:17:00.001856Z", - "iopub.status.busy": "2024-07-25T06:17:00.001763Z", - "iopub.status.idle": "2024-07-25T06:17:00.270101Z", - "shell.execute_reply": "2024-07-25T06:17:00.269792Z" + "iopub.execute_input": "2024-11-05T04:49:41.480848Z", + "iopub.status.busy": "2024-11-05T04:49:41.480753Z", + "iopub.status.idle": "2024-11-05T04:49:41.764373Z", + "shell.execute_reply": "2024-11-05T04:49:41.764047Z" } }, "outputs": [], @@ -40,10 +40,10 @@ "id": "29af6ee3", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:17:00.272427Z", - "iopub.status.busy": "2024-07-25T06:17:00.272277Z", - "iopub.status.idle": "2024-07-25T06:17:00.279931Z", - "shell.execute_reply": "2024-07-25T06:17:00.279670Z" + "iopub.execute_input": "2024-11-05T04:49:41.767394Z", + "iopub.status.busy": "2024-11-05T04:49:41.767226Z", + "iopub.status.idle": "2024-11-05T04:49:41.776312Z", + "shell.execute_reply": "2024-11-05T04:49:41.775962Z" } }, "outputs": [ @@ -67,7 +67,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -104,7 +104,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0" + "version": "3.10.13" } }, "nbformat": 4, diff --git a/docs/api/step-decorators/environment.ipynb b/docs/api/step-decorators/environment.ipynb index c0aa77aa..5299100b 100644 --- a/docs/api/step-decorators/environment.ipynb +++ b/docs/api/step-decorators/environment.ipynb @@ -24,10 +24,10 @@ "id": "627487a4", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:17:03.511324Z", - "iopub.status.busy": "2024-07-25T06:17:03.510816Z", - "iopub.status.idle": "2024-07-25T06:17:03.798763Z", - "shell.execute_reply": "2024-07-25T06:17:03.798427Z" + "iopub.execute_input": "2024-11-05T04:49:44.631238Z", + "iopub.status.busy": "2024-11-05T04:49:44.631127Z", + "iopub.status.idle": "2024-11-05T04:49:44.910489Z", + "shell.execute_reply": "2024-11-05T04:49:44.910128Z" } }, "outputs": [], @@ -46,10 +46,10 @@ "id": "af50a6d2", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:17:03.800957Z", - "iopub.status.busy": "2024-07-25T06:17:03.800802Z", - "iopub.status.idle": "2024-07-25T06:17:03.805689Z", - "shell.execute_reply": "2024-07-25T06:17:03.805444Z" + "iopub.execute_input": "2024-11-05T04:49:44.921719Z", + "iopub.status.busy": "2024-11-05T04:49:44.915068Z", + "iopub.status.idle": "2024-11-05T04:49:44.928343Z", + "shell.execute_reply": "2024-11-05T04:49:44.928101Z" } }, "outputs": [ @@ -70,7 +70,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -107,7 +107,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0" + "version": "3.10.13" } }, "nbformat": 4, diff --git a/docs/api/step-decorators/kubernetes.ipynb b/docs/api/step-decorators/kubernetes.ipynb index 2ec23e28..b36f03e5 100644 --- a/docs/api/step-decorators/kubernetes.ipynb +++ b/docs/api/step-decorators/kubernetes.ipynb @@ -18,10 +18,10 @@ "id": "e5133463", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:17:04.019492Z", - "iopub.status.busy": "2024-07-25T06:17:04.019392Z", - "iopub.status.idle": "2024-07-25T06:17:04.308718Z", - "shell.execute_reply": "2024-07-25T06:17:04.308378Z" + "iopub.execute_input": "2024-11-05T04:49:45.388766Z", + "iopub.status.busy": "2024-11-05T04:49:45.388671Z", + "iopub.status.idle": "2024-11-05T04:49:45.665844Z", + "shell.execute_reply": "2024-11-05T04:49:45.665436Z" } }, "outputs": [], @@ -40,10 +40,10 @@ "id": "d7399f58", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:17:04.311427Z", - "iopub.status.busy": "2024-07-25T06:17:04.311186Z", - "iopub.status.idle": "2024-07-25T06:17:04.322020Z", - "shell.execute_reply": "2024-07-25T06:17:04.321710Z" + "iopub.execute_input": "2024-11-05T04:49:45.668997Z", + "iopub.status.busy": "2024-11-05T04:49:45.668754Z", + "iopub.status.idle": "2024-11-05T04:49:45.681232Z", + "shell.execute_reply": "2024-11-05T04:49:45.680965Z" } }, "outputs": [ @@ -51,7 +51,7 @@ "data": { "text/html": [ "\n", - "

decorator @kubernetes (...)[source]

metaflow

Specifies that this step should execute on Kubernetes.

Parameters
----------
cpu : int, default 1
    Number of CPUs required for this step. If `@resources` is
    also present, the maximum value from all decorators is used.
memory : int, default 4096
    Memory size (in MB) required for this step. If
    `@resources` is also present, the maximum value from all decorators is
    used.
disk : int, default 10240
    Disk size (in MB) required for this step. If
    `@resources` is also present, the maximum value from all decorators is
    used.
image : str, optional, default None
    Docker image to use when launching on Kubernetes. If not specified, and
    METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
    not, a default Docker image mapping to the current version of Python is used.
image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
    If given, the imagePullPolicy to be applied to the Docker image of the step.
service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
    Kubernetes service account to use when launching pod in Kubernetes.
secrets : List[str], optional, default None
    Kubernetes secrets to use when launching pod in Kubernetes. These
    secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
    in Metaflow configuration.
namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
    Kubernetes namespace to use when launching pod in Kubernetes.
gpu : int, optional, default None
    Number of GPUs required for this step. A value of zero implies that
    the scheduled node should not have GPUs.
gpu_vendor : str, default KUBERNETES_GPU_VENDOR
    The vendor of the GPUs to be used for this step.
tolerations : List[str], default []
    The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
    Kubernetes tolerations to use when launching pod in Kubernetes.
use_tmpfs : bool, default False
    This enables an explicit tmpfs mount for this step.
tmpfs_tempdir : bool, default True
    sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
tmpfs_size : int, optional, default: None
    The value for the size (in MiB) of the tmpfs mount for this step.
    This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
    memory allocated for this step.
tmpfs_path : str, optional, default /metaflow_temp
    Path to tmpfs mount for this step.
persistent_volume_claims : Dict[str, str], optional, default None
    A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
    volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
shared_memory: int, optional
    Shared memory size (in MiB) required for this step
port: int, optional
    Port number to specify in the Kubernetes job object

\n", + "

decorator @kubernetes (...)[source]

metaflow

Specifies that this step should execute on Kubernetes.

Parameters
----------
cpu : int, default 1
    Number of CPUs required for this step. If `@resources` is
    also present, the maximum value from all decorators is used.
memory : int, default 4096
    Memory size (in MB) required for this step. If
    `@resources` is also present, the maximum value from all decorators is
    used.
disk : int, default 10240
    Disk size (in MB) required for this step. If
    `@resources` is also present, the maximum value from all decorators is
    used.
image : str, optional, default None
    Docker image to use when launching on Kubernetes. If not specified, and
    METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
    not, a default Docker image mapping to the current version of Python is used.
image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
    If given, the imagePullPolicy to be applied to the Docker image of the step.
service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
    Kubernetes service account to use when launching pod in Kubernetes.
secrets : List[str], optional, default None
    Kubernetes secrets to use when launching pod in Kubernetes. These
    secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
    in Metaflow configuration.
node_selector: Union[Dict[str,str], str], optional, default None
    Kubernetes node selector(s) to apply to the pod running the task.
    Can be passed in as a comma separated string of values e.g.
    'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
    {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
    Kubernetes namespace to use when launching pod in Kubernetes.
gpu : int, optional, default None
    Number of GPUs required for this step. A value of zero implies that
    the scheduled node should not have GPUs.
gpu_vendor : str, default KUBERNETES_GPU_VENDOR
    The vendor of the GPUs to be used for this step.
tolerations : List[str], default []
    The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
    Kubernetes tolerations to use when launching pod in Kubernetes.
use_tmpfs : bool, default False
    This enables an explicit tmpfs mount for this step.
tmpfs_tempdir : bool, default True
    sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
tmpfs_size : int, optional, default: None
    The value for the size (in MiB) of the tmpfs mount for this step.
    This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
    memory allocated for this step.
tmpfs_path : str, optional, default /metaflow_temp
    Path to tmpfs mount for this step.
persistent_volume_claims : Dict[str, str], optional, default None
    A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
    volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
shared_memory: int, optional
    Shared memory size (in MiB) required for this step
port: int, optional
    Port number to specify in the Kubernetes job object
compute_pool : str, optional, default None
    Compute pool to be used for for this step.
    If not specified, any accessible compute pool within the perimeter is used.
hostname_resolution_timeout: int, default 10 * 60
    Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
    Only applicable when @parallel is used.

\n", "\n", "\n", "\n", @@ -66,6 +66,7 @@ "\t\n", "\t\n", "\t\n", + "\t\n", "\t\n", "\t\n", "\t\n", @@ -77,11 +78,13 @@ "\t\n", "\t\n", "\t\n", + "\t\n", + "\t\n", "\n", "" ], "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -118,7 +121,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0" + "version": "3.10.13" } }, "nbformat": 4, diff --git a/docs/api/step-decorators/kubernetes.md b/docs/api/step-decorators/kubernetes.md index c62cdd74..3d388bae 100644 --- a/docs/api/step-decorators/kubernetes.md +++ b/docs/api/step-decorators/kubernetes.md @@ -20,6 +20,7 @@ For options related to `tmpfs`, see [Using `metaflow.S3` for in-memory processin + @@ -31,6 +32,8 @@ For options related to `tmpfs`, see [Using `metaflow.S3` for in-memory processin + + diff --git a/docs/api/step-decorators/pypi.ipynb b/docs/api/step-decorators/pypi.ipynb index bb18fed5..7b11866d 100644 --- a/docs/api/step-decorators/pypi.ipynb +++ b/docs/api/step-decorators/pypi.ipynb @@ -18,10 +18,10 @@ "id": "8d5bb116", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:17:02.673485Z", - "iopub.status.busy": "2024-07-25T06:17:02.673407Z", - "iopub.status.idle": "2024-07-25T06:17:03.004750Z", - "shell.execute_reply": "2024-07-25T06:17:03.004331Z" + "iopub.execute_input": "2024-11-05T04:49:43.669520Z", + "iopub.status.busy": "2024-11-05T04:49:43.669419Z", + "iopub.status.idle": "2024-11-05T04:49:43.963356Z", + "shell.execute_reply": "2024-11-05T04:49:43.963036Z" } }, "outputs": [], @@ -40,10 +40,10 @@ "id": "29af6ee3", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:17:03.009355Z", - "iopub.status.busy": "2024-07-25T06:17:03.009178Z", - "iopub.status.idle": "2024-07-25T06:17:03.015047Z", - "shell.execute_reply": "2024-07-25T06:17:03.014828Z" + "iopub.execute_input": "2024-11-05T04:49:43.966696Z", + "iopub.status.busy": "2024-11-05T04:49:43.966506Z", + "iopub.status.idle": "2024-11-05T04:49:43.973782Z", + "shell.execute_reply": "2024-11-05T04:49:43.973459Z" } }, "outputs": [ @@ -65,7 +65,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -102,7 +102,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0" + "version": "3.10.13" } }, "nbformat": 4, diff --git a/docs/api/step-decorators/resources.ipynb b/docs/api/step-decorators/resources.ipynb index 257e1354..b552c763 100644 --- a/docs/api/step-decorators/resources.ipynb +++ b/docs/api/step-decorators/resources.ipynb @@ -18,10 +18,10 @@ "id": "c8aea2b0", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:58.369242Z", - "iopub.status.busy": "2024-07-25T06:16:58.369161Z", - "iopub.status.idle": "2024-07-25T06:16:58.665405Z", - "shell.execute_reply": "2024-07-25T06:16:58.665060Z" + "iopub.execute_input": "2024-11-05T04:49:39.474127Z", + "iopub.status.busy": "2024-11-05T04:49:39.474004Z", + "iopub.status.idle": "2024-11-05T04:49:39.768595Z", + "shell.execute_reply": "2024-11-05T04:49:39.768256Z" } }, "outputs": [], @@ -40,10 +40,10 @@ "id": "1d16a1c7", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:58.668000Z", - "iopub.status.busy": "2024-07-25T06:16:58.667823Z", - "iopub.status.idle": "2024-07-25T06:16:58.673649Z", - "shell.execute_reply": "2024-07-25T06:16:58.673353Z" + "iopub.execute_input": "2024-11-05T04:49:39.771921Z", + "iopub.status.busy": "2024-11-05T04:49:39.771545Z", + "iopub.status.idle": "2024-11-05T04:49:39.777938Z", + "shell.execute_reply": "2024-11-05T04:49:39.777590Z" } }, "outputs": [ @@ -51,7 +51,7 @@ "data": { "text/html": [ "\n", - "

decorator @resources (...)[source]

metaflow

Specifies the resources needed when executing this step.

Use `@resources` to specify the resource requirements
independently of the specific compute layer (`@batch`, `@kubernetes`).

You can choose the compute layer on the command line by executing e.g.
```
python myflow.py run --with batch
```
or
```
python myflow.py run --with kubernetes
```
which executes the flow on the desired system using the
requirements specified in `@resources`.

Parameters
----------
cpu : int, default 1
    Number of CPUs required for this step.
gpu : int, default 0
    Number of GPUs required for this step.
disk : int, optional, default None
    Disk size (in MB) required for this step. Only applies on Kubernetes.
memory : int, default 4096
    Memory size (in MB) required for this step.
shared_memory : int, optional, default None
    The value for the size (in MiB) of the /dev/shm volume for this step.
    This parameter maps to the `--shm-size` option in Docker.

\n", + "

decorator @resources (...)[source]

metaflow

Specifies the resources needed when executing this step.

Use `@resources` to specify the resource requirements
independently of the specific compute layer (`@batch`, `@kubernetes`).

You can choose the compute layer on the command line by executing e.g.
```
python myflow.py run --with batch
```
or
```
python myflow.py run --with kubernetes
```
which executes the flow on the desired system using the
requirements specified in `@resources`.

Parameters
----------
cpu : int, default 1
    Number of CPUs required for this step.
gpu : int, optional, default None
    Number of GPUs required for this step.
disk : int, optional, default None
    Disk size (in MB) required for this step. Only applies on Kubernetes.
memory : int, default 4096
    Memory size (in MB) required for this step.
shared_memory : int, optional, default None
    The value for the size (in MiB) of the /dev/shm volume for this step.
    This parameter maps to the `--shm-size` option in Docker.

\n", "\n", "\n", "\n", @@ -60,7 +60,7 @@ "\n", "\n", "\t\n", - "\t\n", + "\t\n", "\t\n", "\t\n", "\t\n", @@ -68,7 +68,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -105,7 +105,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0" + "version": "3.10.13" } }, "nbformat": 4, diff --git a/docs/api/step-decorators/resources.md b/docs/api/step-decorators/resources.md index 820ec44e..715cf6c3 100644 --- a/docs/api/step-decorators/resources.md +++ b/docs/api/step-decorators/resources.md @@ -14,7 +14,7 @@ Note that `@resources` takes effect only when combined with another decorator li - + diff --git a/docs/api/step-decorators/retry.ipynb b/docs/api/step-decorators/retry.ipynb index 79cca018..def89fa9 100644 --- a/docs/api/step-decorators/retry.ipynb +++ b/docs/api/step-decorators/retry.ipynb @@ -18,10 +18,10 @@ "id": "627487a4", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:59.552838Z", - "iopub.status.busy": "2024-07-25T06:16:59.552756Z", - "iopub.status.idle": "2024-07-25T06:16:59.832917Z", - "shell.execute_reply": "2024-07-25T06:16:59.832549Z" + "iopub.execute_input": "2024-11-05T04:49:40.978454Z", + "iopub.status.busy": "2024-11-05T04:49:40.978322Z", + "iopub.status.idle": "2024-11-05T04:49:41.269776Z", + "shell.execute_reply": "2024-11-05T04:49:41.269450Z" } }, "outputs": [], @@ -40,10 +40,10 @@ "id": "af50a6d2", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:16:59.835195Z", - "iopub.status.busy": "2024-07-25T06:16:59.835052Z", - "iopub.status.idle": "2024-07-25T06:16:59.840908Z", - "shell.execute_reply": "2024-07-25T06:16:59.840631Z" + "iopub.execute_input": "2024-11-05T04:49:41.273139Z", + "iopub.status.busy": "2024-11-05T04:49:41.272942Z", + "iopub.status.idle": "2024-11-05T04:49:41.278607Z", + "shell.execute_reply": "2024-11-05T04:49:41.278380Z" } }, "outputs": [ @@ -65,7 +65,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -102,7 +102,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0" + "version": "3.10.13" } }, "nbformat": 4, diff --git a/docs/api/step-decorators/secrets.ipynb b/docs/api/step-decorators/secrets.ipynb index fe87e2d3..fe6787d4 100644 --- a/docs/api/step-decorators/secrets.ipynb +++ b/docs/api/step-decorators/secrets.ipynb @@ -16,10 +16,10 @@ "id": "627487a4", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:17:01.505025Z", - "iopub.status.busy": "2024-07-25T06:17:01.504912Z", - "iopub.status.idle": "2024-07-25T06:17:02.122202Z", - "shell.execute_reply": "2024-07-25T06:17:02.121862Z" + "iopub.execute_input": "2024-11-05T04:49:42.667652Z", + "iopub.status.busy": "2024-11-05T04:49:42.667562Z", + "iopub.status.idle": "2024-11-05T04:49:42.962842Z", + "shell.execute_reply": "2024-11-05T04:49:42.962423Z" } }, "outputs": [], @@ -38,10 +38,10 @@ "id": "af50a6d2", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:17:02.126004Z", - "iopub.status.busy": "2024-07-25T06:17:02.125801Z", - "iopub.status.idle": "2024-07-25T06:17:02.133368Z", - "shell.execute_reply": "2024-07-25T06:17:02.132965Z" + "iopub.execute_input": "2024-11-05T04:49:42.966041Z", + "iopub.status.busy": "2024-11-05T04:49:42.965822Z", + "iopub.status.idle": "2024-11-05T04:49:42.973715Z", + "shell.execute_reply": "2024-11-05T04:49:42.973388Z" } }, "outputs": [ @@ -62,7 +62,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -99,7 +99,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0" + "version": "3.10.13" } }, "nbformat": 4, diff --git a/docs/api/step-decorators/step.ipynb b/docs/api/step-decorators/step.ipynb index 049cd566..9c61584e 100644 --- a/docs/api/step-decorators/step.ipynb +++ b/docs/api/step-decorators/step.ipynb @@ -18,10 +18,10 @@ "id": "627487a4", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:17:00.752153Z", - "iopub.status.busy": "2024-07-25T06:17:00.752043Z", - "iopub.status.idle": "2024-07-25T06:17:01.014513Z", - "shell.execute_reply": "2024-07-25T06:17:01.014212Z" + "iopub.execute_input": "2024-11-05T04:49:41.673320Z", + "iopub.status.busy": "2024-11-05T04:49:41.673180Z", + "iopub.status.idle": "2024-11-05T04:49:41.967655Z", + "shell.execute_reply": "2024-11-05T04:49:41.966880Z" } }, "outputs": [], @@ -40,10 +40,10 @@ "id": "af50a6d2", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:17:01.017025Z", - "iopub.status.busy": "2024-07-25T06:17:01.016865Z", - "iopub.status.idle": "2024-07-25T06:17:01.021112Z", - "shell.execute_reply": "2024-07-25T06:17:01.020885Z" + "iopub.execute_input": "2024-11-05T04:49:41.970863Z", + "iopub.status.busy": "2024-11-05T04:49:41.970615Z", + "iopub.status.idle": "2024-11-05T04:49:41.976739Z", + "shell.execute_reply": "2024-11-05T04:49:41.976329Z" } }, "outputs": [ @@ -51,9 +51,9 @@ "data": { "text/html": [ "\n", - "

function step [source]

metaflow

Marks a method in a FlowSpec as a Metaflow Step. Note that this
decorator needs to be placed as close to the method as possible (ie:
before other decorators).

In other words, this is valid:
```
@batch
@step
def foo(self):
    pass
```

whereas this is not:
```
@step
@batch
def foo(self):
    pass
```

Parameters
----------
f : Union[Callable[[FlowSpecDerived], None], Callable[[FlowSpecDerived, Any], None]]
    Function to make into a Metaflow Step

Returns
-------
Union[Callable[[FlowSpecDerived, StepFlag], None], Callable[[FlowSpecDerived, Any, StepFlag], None]]
    Function that is a Metaflow Step

\n", + "

function step [source]

metaflow

Marks a method in a FlowSpec as a Metaflow Step. Note that this
decorator needs to be placed as close to the method as possible (ie:
before other decorators).

In other words, this is valid:
```
@batch
@step
def foo(self):
    pass
```

whereas this is not:
```
@step
@batch
def foo(self):
    pass
```

Parameters
----------
f : Union[Callable[[FlowSpecDerived], None], Callable[[FlowSpecDerived, Any], None]]
    Function to make into a Metaflow Step

Returns
-------
Union[Callable[[FlowSpecDerived, StepFlag], None], Callable[[FlowSpecDerived, Any, StepFlag], None]]
    Function that is a Metaflow Step

\n", "\n", - "\n", + "\n", "\n", "\n", "\n", @@ -67,7 +67,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -104,7 +104,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0" + "version": "3.10.13" } }, "nbformat": 4, diff --git a/docs/api/step-decorators/step.md b/docs/api/step-decorators/step.md index 87f50c39..acdff4c3 100644 --- a/docs/api/step-decorators/step.md +++ b/docs/api/step-decorators/step.md @@ -7,7 +7,7 @@ Use `@step` to construct Metaflow workflows. For more information, see [Basics o - + diff --git a/docs/api/step-decorators/timeout.ipynb b/docs/api/step-decorators/timeout.ipynb index f57fd574..15b2df23 100644 --- a/docs/api/step-decorators/timeout.ipynb +++ b/docs/api/step-decorators/timeout.ipynb @@ -18,10 +18,10 @@ "id": "627487a4", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:17:03.018541Z", - "iopub.status.busy": "2024-07-25T06:17:03.018463Z", - "iopub.status.idle": "2024-07-25T06:17:03.292350Z", - "shell.execute_reply": "2024-07-25T06:17:03.292035Z" + "iopub.execute_input": "2024-11-05T04:49:44.386067Z", + "iopub.status.busy": "2024-11-05T04:49:44.385950Z", + "iopub.status.idle": "2024-11-05T04:49:44.658942Z", + "shell.execute_reply": "2024-11-05T04:49:44.658553Z" } }, "outputs": [], @@ -40,10 +40,10 @@ "id": "af50a6d2", "metadata": { "execution": { - "iopub.execute_input": "2024-07-25T06:17:03.294731Z", - "iopub.status.busy": "2024-07-25T06:17:03.294553Z", - "iopub.status.idle": "2024-07-25T06:17:03.300684Z", - "shell.execute_reply": "2024-07-25T06:17:03.300430Z" + "iopub.execute_input": "2024-11-05T04:49:44.662232Z", + "iopub.status.busy": "2024-11-05T04:49:44.662042Z", + "iopub.status.idle": "2024-11-05T04:49:44.668610Z", + "shell.execute_reply": "2024-11-05T04:49:44.668335Z" } }, "outputs": [ @@ -66,7 +66,7 @@ "" ], "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -103,7 +103,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0" + "version": "3.10.13" } }, "nbformat": 4,