mirror of
https://github.com/Lightning-AI/pytorch-lightning.git
synced 2026-10-12 02:06:43 +00:00
Remove NeptuneLogger integration from PyTorch Lightning (#21572)
* Remove NeptuneLogger integration from PyTorch Lightning - Deleted NeptuneLogger from the logger initialization and related files. - Updated documentation and tests to reflect the removal of NeptuneLogger. - Adjusted logger-related code to ensure compatibility without NeptuneLogger. * keep `NeptuneLogger` class as a wrapper and raise an error suggesting litlogger instead * update * update * Update src/lightning/pytorch/loggers/neptune.py --------- Co-authored-by: Bhimraj Yadav <[email protected]> Co-authored-by: Justus Schock <[email protected]>
This commit is contained in:
Deependu
co-authored by
Bhimraj Yadav
Justus Schock
Bhimraj Yadav
Justus Schock
parent
1690c215ad
commit
73f593d3bb
@@ -8,12 +8,6 @@
|
||||
},
|
||||
{
|
||||
"pattern": "^https://codecov.io/gh/Lightning-AI/pytorch-lightning/graph/badge.svg"
|
||||
},
|
||||
{
|
||||
"pattern": "^https://app.neptune.ai"
|
||||
},
|
||||
{
|
||||
"pattern": "^https://www.neptune.ai/"
|
||||
}
|
||||
],
|
||||
"httpHeaders": [
|
||||
|
||||
@@ -169,7 +169,6 @@ wandb
|
||||
.forked/
|
||||
*.prof
|
||||
*.tar.gz
|
||||
.neptune/
|
||||
|
||||
# dataset generated from bolts in examples.
|
||||
cifar-10-batches-py
|
||||
|
||||
@@ -292,9 +292,6 @@ trainer = Trainer(logger=loggers.CometLogger())
|
||||
# mlflow
|
||||
trainer = Trainer(logger=loggers.MLFlowLogger())
|
||||
|
||||
# neptune
|
||||
trainer = Trainer(logger=loggers.NeptuneLogger())
|
||||
|
||||
# ... and dozens more
|
||||
```
|
||||
|
||||
|
||||
@@ -99,7 +99,6 @@ loggers
|
||||
comet
|
||||
csv_logs
|
||||
mlflow
|
||||
neptune
|
||||
tensorboard
|
||||
wandb
|
||||
|
||||
|
||||
@@ -351,7 +351,6 @@ intersphinx_mapping = {
|
||||
"lightning.fabric": ("https://lightning.ai/docs/fabric/stable/", None),
|
||||
# TODO: these are missing objects.inv
|
||||
# "comet_ml": ("https://www.comet.com/docs/v2/", None),
|
||||
# "neptune": ("https://docs.neptune.ai/", None),
|
||||
# "wandb": ("https://docs.wandb.ai//", None),
|
||||
}
|
||||
nitpicky = True
|
||||
@@ -457,8 +456,6 @@ nitpick_ignore = [
|
||||
("py:class", "mlflow.tracking.MlflowClient"),
|
||||
("py:attr", "model"),
|
||||
("py:meth", "move_data_to_device"),
|
||||
("py:class", "neptune.Run"),
|
||||
("py:class", "neptune.handler.Handler"),
|
||||
("py:meth", "on_after_batch_transfer"),
|
||||
("py:meth", "on_before_batch_transfer"),
|
||||
("py:meth", "on_save_checkpoint"),
|
||||
@@ -605,7 +602,6 @@ from lightning.pytorch.callbacks import Callback
|
||||
from lightning.pytorch.cli import _JSONARGPARSE_SIGNATURES_AVAILABLE as _JSONARGPARSE_AVAILABLE
|
||||
from lightning.pytorch.utilities.imports import _TORCHVISION_AVAILABLE
|
||||
from lightning.fabric.loggers.tensorboard import _TENSORBOARD_AVAILABLE, _TENSORBOARDX_AVAILABLE
|
||||
from lightning.pytorch.loggers.neptune import _NEPTUNE_AVAILABLE
|
||||
from lightning.pytorch.loggers.comet import _COMET_AVAILABLE
|
||||
from lightning.pytorch.loggers.mlflow import _MLFLOW_AVAILABLE
|
||||
from lightning.pytorch.loggers.wandb import _WANDB_AVAILABLE
|
||||
@@ -638,7 +634,4 @@ linkcheck_ignore = [
|
||||
"https://openai.com/blog/.*",
|
||||
"https://openai.com/index/*",
|
||||
"https://tinyurl.com/.*", # has a human verification check on redirect
|
||||
"https://docs.neptune.ai/.*", # TODO: remove after dropping Neptune support
|
||||
"https://app.neptune.ai/*",
|
||||
"https://www.neptune.ai/*"
|
||||
]
|
||||
|
||||
@@ -31,7 +31,6 @@ The following are loggers we support:
|
||||
CometLogger
|
||||
CSVLogger
|
||||
MLFlowLogger
|
||||
NeptuneLogger
|
||||
TensorBoardLogger
|
||||
WandbLogger
|
||||
|
||||
@@ -69,7 +68,7 @@ You can also pass a custom Logger to the :class:`~lightning.pytorch.trainer.trai
|
||||
tb_logger = pl_loggers.TensorBoardLogger(save_dir="logs/")
|
||||
trainer = Trainer(logger=tb_logger)
|
||||
|
||||
Choose from any of the others such as MLflow, Comet, Neptune, WandB, etc.
|
||||
Choose from any of the others such as LitLogger, MLflow, Comet, WandB, etc.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
|
||||
@@ -88,47 +88,6 @@ Here's the full documentation for the :class:`~lightning.pytorch.loggers.MLFlowL
|
||||
|
||||
----
|
||||
|
||||
Neptune.ai
|
||||
==========
|
||||
To use `Neptune.ai <https://www.neptune.ai/>`_ first install the neptune package:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
pip install neptune
|
||||
|
||||
or with conda:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
conda install -c conda-forge neptune
|
||||
|
||||
Configure the logger and pass it to the :class:`~lightning.pytorch.trainer.trainer.Trainer`:
|
||||
|
||||
.. testcode::
|
||||
:skipif: not _NEPTUNE_AVAILABLE
|
||||
|
||||
import neptune
|
||||
from lightning.pytorch.loggers import NeptuneLogger
|
||||
|
||||
neptune_logger = NeptuneLogger(
|
||||
api_key=neptune.ANONYMOUS_API_TOKEN, # replace with your own
|
||||
project="common/pytorch-lightning-integration", # format "<WORKSPACE/PROJECT>"
|
||||
)
|
||||
trainer = Trainer(logger=neptune_logger)
|
||||
|
||||
Access the neptune logger from any function (except the LightningModule *init*) to use its API for tracking advanced artifacts
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
class LitModel(LightningModule):
|
||||
def any_lightning_module_function_or_hook(self):
|
||||
neptune_logger = self.logger.experiment["your/metadata/structure"]
|
||||
neptune_logger.append(metadata)
|
||||
|
||||
Here's the full documentation for the :class:`~lightning.pytorch.loggers.NeptuneLogger`.
|
||||
|
||||
----
|
||||
|
||||
Tensorboard
|
||||
===========
|
||||
`TensorBoard <https://pytorch.org/docs/stable/tensorboard.html>`_ can be installed with:
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# all supported loggers. this list is here as a reference, but they are not installed in CI
|
||||
|
||||
litlogger >= 2026.03.17
|
||||
neptune >=1.0.0
|
||||
comet-ml >=3.31.0
|
||||
mlflow >=1.0.0
|
||||
wandb >=0.12.10
|
||||
|
||||
@@ -15,7 +15,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
|
||||
|
||||
### Removed
|
||||
|
||||
-
|
||||
- Removed support for Neptune logger ([#21572](https://github.com/Lightning-AI/pytorch-lightning/pull/21572))
|
||||
|
||||
|
||||
### Changed
|
||||
@@ -27,6 +27,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
|
||||
|
||||
- Fixed `val_check_interval` raising `ValueError` when `limit_val_batches=0` and interval exceeds training batches ([#21560](https://github.com/Lightning-AI/pytorch-lightning/pull/21560))
|
||||
|
||||
- Fixed pkg-resources deprecation issue ([#21538](https://github.com/Lightning-AI/pytorch-lightning/pull/21538))
|
||||
|
||||
- Fixed FSDP mixed precision (`bf16-mixed`, `16-mixed`) initializing model parameters in half precision instead of fp32 ([#21586](https://github.com/Lightning-AI/pytorch-lightning/pull/21586))
|
||||
|
||||
- Fixed `device_mesh` type hint in `FSDPStrategy` to accept a 2-element tuple via the CLI ([#21581](https://github.com/Lightning-AI/pytorch-lightning/pull/21581))
|
||||
|
||||
@@ -229,7 +229,7 @@ class ModelCheckpoint(Checkpoint):
|
||||
... )
|
||||
|
||||
# save epoch and val_loss in name, but specify the formatting yourself (e.g. to avoid problems with Tensorboard
|
||||
# or Neptune, due to the presence of characters like '=' or '/')
|
||||
# or other loggers, due to the presence of characters like '=' or '/')
|
||||
# saves a file like: my/path/sample-mnist-epoch02-val_loss0.32.ckpt
|
||||
>>> checkpoint_callback = ModelCheckpoint(
|
||||
... monitor='val/loss',
|
||||
|
||||
@@ -16,7 +16,6 @@ from lightning.pytorch.loggers.csv_logs import CSVLogger
|
||||
from lightning.pytorch.loggers.litlogger import LitLogger
|
||||
from lightning.pytorch.loggers.logger import Logger
|
||||
from lightning.pytorch.loggers.mlflow import MLFlowLogger
|
||||
from lightning.pytorch.loggers.neptune import NeptuneLogger
|
||||
from lightning.pytorch.loggers.tensorboard import TensorBoardLogger
|
||||
from lightning.pytorch.loggers.wandb import WandbLogger
|
||||
|
||||
@@ -28,5 +27,4 @@ __all__ = [
|
||||
"MLFlowLogger",
|
||||
"TensorBoardLogger",
|
||||
"WandbLogger",
|
||||
"NeptuneLogger",
|
||||
]
|
||||
|
||||
@@ -12,583 +12,34 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
Neptune Logger
|
||||
--------------
|
||||
Neptune Logger (Deprecated)
|
||||
---------------------------
|
||||
|
||||
Neptune has been sunset. Please migrate to LitLogger.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import logging
|
||||
import os
|
||||
from argparse import Namespace
|
||||
from collections.abc import Generator
|
||||
from functools import wraps
|
||||
from typing import TYPE_CHECKING, Any, Callable, Optional, Union
|
||||
|
||||
from lightning_utilities.core.imports import RequirementCache
|
||||
from torch import Tensor
|
||||
from typing_extensions import override
|
||||
|
||||
import lightning.pytorch as pl
|
||||
from lightning.fabric.utilities.logger import _add_prefix, _convert_params, _sanitize_callable_params
|
||||
from lightning.pytorch.callbacks import Checkpoint
|
||||
from lightning.pytorch.loggers.logger import Logger, rank_zero_experiment
|
||||
from lightning.pytorch.utilities.model_summary import ModelSummary
|
||||
from lightning.pytorch.utilities.rank_zero import rank_zero_only
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from neptune import Run
|
||||
from neptune.handler import Handler
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Neptune is available with two names on PyPI : `neptune` and `neptune-client`
|
||||
# `neptune` was introduced as a name transition of neptune-client and the long-term target is to get
|
||||
# rid of Neptune-client package completely someday. It was introduced as a part of breaking-changes with a release
|
||||
# of neptune-client==1.0. neptune-client>=1.0 is just an alias of neptune package and have some breaking-changes
|
||||
# in compare to neptune-client<1.0.0.
|
||||
_NEPTUNE_AVAILABLE = RequirementCache("neptune>=1.0")
|
||||
_INTEGRATION_VERSION_KEY = "source_code/integrations/pytorch-lightning"
|
||||
from typing import Any
|
||||
|
||||
|
||||
# Neptune client throws `InactiveRunException` when trying to log to an inactive run.
|
||||
# This may happen when the run was stopped through the UI and the logger is still trying to log to it.
|
||||
def _catch_inactive(func: Callable) -> Callable:
|
||||
@wraps(func)
|
||||
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
||||
from neptune.exceptions import InactiveRunException
|
||||
class NeptuneLogger:
|
||||
"""
|
||||
.. deprecated:: Use :class:`LitLogger` instead.
|
||||
|
||||
with contextlib.suppress(InactiveRunException):
|
||||
return func(*args, **kwargs)
|
||||
Neptune has been sunset and this logger is no longer maintained.
|
||||
Please migrate to **LitLogger**, the recommended logging solution for
|
||||
tracking AI experiments, metrics, inputs, outputs, and artifacts.
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
class NeptuneLogger(Logger):
|
||||
r"""Log using `Neptune <https://docs.neptune.ai/integrations/lightning/>`_.
|
||||
|
||||
Install it with pip:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
pip install neptune
|
||||
|
||||
or conda:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
conda install -c conda-forge neptune-client
|
||||
|
||||
**Quickstart**
|
||||
|
||||
Pass a NeptuneLogger instance to the Trainer to log metadata with Neptune:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
|
||||
from lightning.pytorch import Trainer
|
||||
from lightning.pytorch.loggers import NeptuneLogger
|
||||
import neptune
|
||||
|
||||
neptune_logger = NeptuneLogger(
|
||||
api_key=neptune.ANONYMOUS_API_TOKEN, # replace with your own
|
||||
project="common/pytorch-lightning-integration", # format "workspace-name/project-name"
|
||||
tags=["training", "resnet"], # optional
|
||||
)
|
||||
trainer = Trainer(max_epochs=10, logger=neptune_logger)
|
||||
|
||||
**How to use NeptuneLogger?**
|
||||
|
||||
Use the logger anywhere in your :class:`~lightning.pytorch.core.LightningModule` as follows:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from neptune.types import File
|
||||
from lightning.pytorch import LightningModule
|
||||
|
||||
|
||||
class LitModel(LightningModule):
|
||||
def training_step(self, batch, batch_idx):
|
||||
# log metrics
|
||||
acc = ...
|
||||
self.append("train/loss", loss)
|
||||
|
||||
def any_lightning_module_function_or_hook(self):
|
||||
# log images
|
||||
img = ...
|
||||
self.logger.experiment["train/misclassified_images"].append(File.as_image(img))
|
||||
|
||||
# generic recipe
|
||||
metadata = ...
|
||||
self.logger.experiment["your/metadata/structure"] = metadata
|
||||
|
||||
Note that the syntax ``self.logger.experiment["your/metadata/structure"].append(metadata)`` is specific to
|
||||
Neptune and extends the logger capabilities. It lets you log various types of metadata, such as
|
||||
scores, files, images, interactive visuals, and CSVs.
|
||||
Refer to the `Neptune docs <https://docs.neptune.ai/logging/methods>`_
|
||||
for details.
|
||||
You can also use the regular logger methods ``log_metrics()``, and ``log_hyperparams()`` with NeptuneLogger.
|
||||
|
||||
**Log after fitting or testing is finished**
|
||||
|
||||
You can log objects after the fitting or testing methods are finished:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
neptune_logger = NeptuneLogger(project="common/pytorch-lightning-integration")
|
||||
|
||||
trainer = pl.Trainer(logger=neptune_logger)
|
||||
model = ...
|
||||
datamodule = ...
|
||||
trainer.fit(model, datamodule=datamodule)
|
||||
trainer.test(model, datamodule=datamodule)
|
||||
|
||||
# Log objects after `fit` or `test` methods
|
||||
# model summary
|
||||
neptune_logger.log_model_summary(model=model, max_depth=-1)
|
||||
|
||||
# generic recipe
|
||||
metadata = ...
|
||||
neptune_logger.experiment["your/metadata/structure"] = metadata
|
||||
|
||||
**Log model checkpoints**
|
||||
|
||||
If you have :class:`~lightning.pytorch.callbacks.ModelCheckpoint` configured,
|
||||
the Neptune logger automatically logs model checkpoints.
|
||||
Model weights will be uploaded to the "model/checkpoints" namespace in the Neptune run.
|
||||
You can disable this option with:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
neptune_logger = NeptuneLogger(log_model_checkpoints=False)
|
||||
|
||||
**Pass additional parameters to the Neptune run**
|
||||
|
||||
You can also pass ``neptune_run_kwargs`` to add details to the run, like ``tags`` or ``description``:
|
||||
|
||||
.. testcode::
|
||||
:skipif: not _NEPTUNE_AVAILABLE
|
||||
|
||||
from lightning.pytorch import Trainer
|
||||
from lightning.pytorch.loggers import NeptuneLogger
|
||||
|
||||
neptune_logger = NeptuneLogger(
|
||||
project="common/pytorch-lightning-integration",
|
||||
name="lightning-run",
|
||||
description="mlp quick run with pytorch-lightning",
|
||||
tags=["mlp", "quick-run"],
|
||||
)
|
||||
trainer = Trainer(max_epochs=3, logger=neptune_logger)
|
||||
|
||||
Check `run documentation <https://docs.neptune.ai/api/neptune/#init_run>`_
|
||||
for more info about additional run parameters.
|
||||
|
||||
**Details about Neptune run structure**
|
||||
|
||||
Runs can be viewed as nested dictionary-like structures that you can define in your code.
|
||||
Thanks to this you can easily organize your metadata in a way that is most convenient for you.
|
||||
|
||||
The hierarchical structure that you apply to your metadata is reflected in the Neptune web app.
|
||||
|
||||
See also:
|
||||
- Read about
|
||||
`what objects you can log to Neptune <https://docs.neptune.ai/logging/what_you_can_log/>`_.
|
||||
- Check out an `example run <https://app.neptune.ai/o/common/org/pytorch-lightning-integration/e/PTL-1/all>`_
|
||||
with multiple types of metadata logged.
|
||||
- For more detailed examples, see the
|
||||
`user guide <https://docs.neptune.ai/integrations/lightning/>`_.
|
||||
|
||||
Args:
|
||||
api_key: Optional.
|
||||
Neptune API token, found on https://www.neptune.ai upon registration.
|
||||
You should save your token to the `NEPTUNE_API_TOKEN`
|
||||
environment variable and leave the api_key argument out of your code.
|
||||
Instructions: `Setting your API token <https://docs.neptune.ai/setup/setting_api_token/>`_.
|
||||
project: Optional.
|
||||
Name of a project in the form "workspace-name/project-name", for example "tom/mask-rcnn".
|
||||
If ``None``, the value of `NEPTUNE_PROJECT` environment variable is used.
|
||||
You need to create the project on https://www.neptune.ai first.
|
||||
name: Optional. Editable name of the run.
|
||||
The run name is displayed in the Neptune web app.
|
||||
run: Optional. Default is ``None``. A Neptune ``Run`` object.
|
||||
If specified, this existing run will be used for logging, instead of a new run being created.
|
||||
You can also pass a namespace handler object; for example, ``run["test"]``, in which case all
|
||||
metadata is logged under the "test" namespace inside the run.
|
||||
log_model_checkpoints: Optional. Default is ``True``. Log model checkpoint to Neptune.
|
||||
Works only if ``ModelCheckpoint`` is passed to the ``Trainer``.
|
||||
prefix: Optional. Default is ``"training"``. Root namespace for all metadata logging.
|
||||
\**neptune_run_kwargs: Additional arguments like ``tags``, ``description``, ``capture_stdout``, etc.
|
||||
used when a run is created.
|
||||
See: https://github.com/Lightning-AI/LitLogger
|
||||
|
||||
Raises:
|
||||
ModuleNotFoundError:
|
||||
If the required Neptune package is not installed.
|
||||
ValueError:
|
||||
If an argument passed to the logger's constructor is incorrect.
|
||||
|
||||
RuntimeError: Always raised when attempting to instantiate this logger.
|
||||
"""
|
||||
|
||||
LOGGER_JOIN_CHAR = "/"
|
||||
PARAMETERS_KEY = "hyperparams"
|
||||
ARTIFACTS_KEY = "artifacts"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*, # force users to call `NeptuneLogger` initializer with `kwargs`
|
||||
api_key: Optional[str] = None,
|
||||
project: Optional[str] = None,
|
||||
name: Optional[str] = None,
|
||||
run: Optional[Union["Run", "Handler"]] = None,
|
||||
log_model_checkpoints: Optional[bool] = True,
|
||||
prefix: str = "training",
|
||||
**neptune_run_kwargs: Any,
|
||||
):
|
||||
if not _NEPTUNE_AVAILABLE:
|
||||
raise ModuleNotFoundError(str(_NEPTUNE_AVAILABLE))
|
||||
|
||||
# verify if user passed proper init arguments
|
||||
self._verify_input_arguments(api_key, project, name, run, neptune_run_kwargs)
|
||||
super().__init__()
|
||||
self._log_model_checkpoints = log_model_checkpoints
|
||||
self._prefix = prefix
|
||||
self._run_name = name
|
||||
self._project_name = project
|
||||
self._api_key = api_key
|
||||
self._run_instance = run
|
||||
self._neptune_run_kwargs = neptune_run_kwargs
|
||||
self._run_short_id: Optional[str] = None
|
||||
|
||||
if self._run_instance is not None:
|
||||
self._retrieve_run_data()
|
||||
|
||||
from neptune.handler import Handler
|
||||
|
||||
# make sure that we've log integration version for outside `Run` instances
|
||||
root_obj = self._run_instance
|
||||
if isinstance(root_obj, Handler):
|
||||
root_obj = root_obj.get_root_object()
|
||||
|
||||
root_obj[_INTEGRATION_VERSION_KEY] = pl.__version__
|
||||
|
||||
def _retrieve_run_data(self) -> None:
|
||||
from neptune.handler import Handler
|
||||
|
||||
assert self._run_instance is not None
|
||||
root_obj = self._run_instance
|
||||
if isinstance(root_obj, Handler):
|
||||
root_obj = root_obj.get_root_object()
|
||||
|
||||
root_obj.wait()
|
||||
|
||||
if root_obj.exists("sys/id"):
|
||||
self._run_short_id = root_obj["sys/id"].fetch()
|
||||
self._run_name = root_obj["sys/name"].fetch()
|
||||
else:
|
||||
self._run_short_id = "OFFLINE"
|
||||
self._run_name = "offline-name"
|
||||
|
||||
@property
|
||||
def _neptune_init_args(self) -> dict:
|
||||
args: dict = {}
|
||||
# Backward compatibility in case of previous version retrieval
|
||||
with contextlib.suppress(AttributeError):
|
||||
args = self._neptune_run_kwargs
|
||||
|
||||
if self._project_name is not None:
|
||||
args["project"] = self._project_name
|
||||
|
||||
if self._api_key is not None:
|
||||
args["api_token"] = self._api_key
|
||||
|
||||
if self._run_short_id is not None:
|
||||
args["run"] = self._run_short_id
|
||||
|
||||
# Backward compatibility in case of previous version retrieval
|
||||
with contextlib.suppress(AttributeError):
|
||||
if self._run_name is not None:
|
||||
args["name"] = self._run_name
|
||||
|
||||
return args
|
||||
|
||||
def _construct_path_with_prefix(self, *keys: str) -> str:
|
||||
"""Return sequence of keys joined by `LOGGER_JOIN_CHAR`, started with `_prefix` if defined."""
|
||||
if self._prefix:
|
||||
return self.LOGGER_JOIN_CHAR.join([self._prefix, *keys])
|
||||
return self.LOGGER_JOIN_CHAR.join(keys)
|
||||
|
||||
@staticmethod
|
||||
def _verify_input_arguments(
|
||||
api_key: Optional[str],
|
||||
project: Optional[str],
|
||||
name: Optional[str],
|
||||
run: Optional[Union["Run", "Handler"]],
|
||||
neptune_run_kwargs: dict,
|
||||
) -> None:
|
||||
from neptune import Run
|
||||
from neptune.handler import Handler
|
||||
|
||||
# check if user passed the client `Run`/`Handler` object
|
||||
if run is not None and not isinstance(run, (Run, Handler)):
|
||||
raise ValueError("Run parameter expected to be of type `neptune.Run`, or `neptune.handler.Handler`.")
|
||||
|
||||
# check if user passed redundant neptune.init_run arguments when passed run
|
||||
any_neptune_init_arg_passed = any(arg is not None for arg in [api_key, project, name]) or neptune_run_kwargs
|
||||
if run is not None and any_neptune_init_arg_passed:
|
||||
raise ValueError(
|
||||
"When an already initialized run object is provided, you can't provide other `neptune.init_run()`"
|
||||
" parameters."
|
||||
)
|
||||
|
||||
def __getstate__(self) -> dict[str, Any]:
|
||||
state = self.__dict__.copy()
|
||||
# Run instance can't be pickled
|
||||
state["_run_instance"] = None
|
||||
return state
|
||||
|
||||
def __setstate__(self, state: dict[str, Any]) -> None:
|
||||
import neptune
|
||||
|
||||
self.__dict__ = state
|
||||
self._run_instance = neptune.init_run(**self._neptune_init_args)
|
||||
|
||||
@property
|
||||
@rank_zero_experiment
|
||||
def experiment(self) -> "Run":
|
||||
r"""Actual Neptune run object. Allows you to use neptune logging features in your
|
||||
:class:`~lightning.pytorch.core.LightningModule`.
|
||||
|
||||
Example::
|
||||
|
||||
class LitModel(LightningModule):
|
||||
def training_step(self, batch, batch_idx):
|
||||
# log metrics
|
||||
acc = ...
|
||||
self.logger.experiment["train/acc"].append(acc)
|
||||
|
||||
# log images
|
||||
img = ...
|
||||
self.logger.experiment["train/misclassified_images"].append(File.as_image(img))
|
||||
|
||||
Note that the syntax ``self.logger.experiment["your/metadata/structure"].append(metadata)``
|
||||
is specific to Neptune and extends the logger capabilities.
|
||||
It lets you log various types of metadata, such as scores, files,
|
||||
images, interactive visuals, and CSVs. Refer to the
|
||||
`Neptune docs <https://docs.neptune.ai/logging/methods>`_
|
||||
for more detailed explanations.
|
||||
You can also use the regular logger methods ``log_metrics()``, and ``log_hyperparams()``
|
||||
with NeptuneLogger.
|
||||
|
||||
"""
|
||||
return self.run
|
||||
|
||||
@property
|
||||
@rank_zero_experiment
|
||||
def run(self) -> "Run":
|
||||
import neptune
|
||||
|
||||
if not self._run_instance:
|
||||
self._run_instance = neptune.init_run(**self._neptune_init_args)
|
||||
self._retrieve_run_data()
|
||||
# make sure that we've log integration version for newly created
|
||||
self._run_instance[_INTEGRATION_VERSION_KEY] = pl.__version__
|
||||
|
||||
return self._run_instance
|
||||
|
||||
@override
|
||||
@rank_zero_only
|
||||
@_catch_inactive
|
||||
def log_hyperparams(self, params: Union[dict[str, Any], Namespace]) -> None:
|
||||
r"""Log hyperparameters to the run.
|
||||
|
||||
Hyperparameters will be logged under the "<prefix>/hyperparams" namespace.
|
||||
|
||||
Note:
|
||||
|
||||
You can also log parameters by directly using the logger instance:
|
||||
``neptune_logger.experiment["model/hyper-parameters"] = params_dict``.
|
||||
|
||||
In this way you can keep hierarchical structure of the parameters.
|
||||
|
||||
Args:
|
||||
params: `dict`.
|
||||
Python dictionary structure with parameters.
|
||||
|
||||
Example::
|
||||
|
||||
from lightning.pytorch.loggers import NeptuneLogger
|
||||
import neptune
|
||||
|
||||
PARAMS = {
|
||||
"batch_size": 64,
|
||||
"lr": 0.07,
|
||||
"decay_factor": 0.97,
|
||||
}
|
||||
|
||||
neptune_logger = NeptuneLogger(
|
||||
api_key=neptune.ANONYMOUS_API_TOKEN,
|
||||
project="common/pytorch-lightning-integration"
|
||||
)
|
||||
|
||||
neptune_logger.log_hyperparams(PARAMS)
|
||||
|
||||
"""
|
||||
from neptune.utils import stringify_unsupported
|
||||
|
||||
params = _convert_params(params)
|
||||
params = _sanitize_callable_params(params)
|
||||
|
||||
parameters_key = self.PARAMETERS_KEY
|
||||
parameters_key = self._construct_path_with_prefix(parameters_key)
|
||||
|
||||
self.run[parameters_key] = stringify_unsupported(params)
|
||||
|
||||
@override
|
||||
@rank_zero_only
|
||||
@_catch_inactive
|
||||
def log_metrics(self, metrics: dict[str, Union[Tensor, float]], step: Optional[int] = None) -> None:
|
||||
"""Log metrics (numeric values) in Neptune runs.
|
||||
|
||||
Args:
|
||||
metrics: Dictionary with metric names as keys and measured quantities as values.
|
||||
step: Step number at which the metrics should be recorded
|
||||
|
||||
"""
|
||||
if rank_zero_only.rank != 0:
|
||||
raise ValueError("run tried to log from global_rank != 0")
|
||||
|
||||
metrics = _add_prefix(metrics, self._prefix, self.LOGGER_JOIN_CHAR)
|
||||
|
||||
for key, val in metrics.items():
|
||||
self.run[key].append(val, step=step)
|
||||
|
||||
@override
|
||||
@rank_zero_only
|
||||
@_catch_inactive
|
||||
def finalize(self, status: str) -> None:
|
||||
if not self._run_instance:
|
||||
# When using multiprocessing, finalize() should be a no-op on the main process, as no experiment has been
|
||||
# initialized there
|
||||
return
|
||||
if status:
|
||||
self.run[self._construct_path_with_prefix("status")] = status
|
||||
|
||||
super().finalize(status)
|
||||
|
||||
@property
|
||||
@override
|
||||
def save_dir(self) -> Optional[str]:
|
||||
"""Gets the save directory of the experiment which in this case is ``None`` because Neptune does not save
|
||||
locally.
|
||||
|
||||
Returns:
|
||||
the root directory where experiment logs get saved
|
||||
|
||||
"""
|
||||
return os.path.join(os.getcwd(), ".neptune")
|
||||
|
||||
@rank_zero_only
|
||||
@_catch_inactive
|
||||
def log_model_summary(self, model: "pl.LightningModule", max_depth: int = -1) -> None:
|
||||
from neptune.types import File
|
||||
|
||||
model_str = str(ModelSummary(model=model, max_depth=max_depth))
|
||||
self.run[self._construct_path_with_prefix("model/summary")] = File.from_content(
|
||||
content=model_str, extension="txt"
|
||||
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
||||
"""Raise an error indicating Neptune logger is no longer supported."""
|
||||
raise RuntimeError(
|
||||
"NeptuneLogger is no longer supported. Neptune has been sunset.\n"
|
||||
"\n"
|
||||
"Please migrate to LitLogger, the recommended logger for AI experiments.\n"
|
||||
"See: https://github.com/lightning-ai/litlogger\n"
|
||||
)
|
||||
|
||||
@override
|
||||
@rank_zero_only
|
||||
@_catch_inactive
|
||||
def after_save_checkpoint(self, checkpoint_callback: Checkpoint) -> None:
|
||||
"""Automatically log checkpointed model. Called after model checkpoint callback saves a new checkpoint.
|
||||
|
||||
Args:
|
||||
checkpoint_callback: the model checkpoint callback instance
|
||||
|
||||
"""
|
||||
if not self._log_model_checkpoints:
|
||||
return
|
||||
|
||||
file_names = set()
|
||||
checkpoints_namespace = self._construct_path_with_prefix("model/checkpoints")
|
||||
|
||||
# save last model
|
||||
if hasattr(checkpoint_callback, "last_model_path") and checkpoint_callback.last_model_path:
|
||||
model_last_name = self._get_full_model_name(checkpoint_callback.last_model_path, checkpoint_callback)
|
||||
file_names.add(model_last_name)
|
||||
self.run[f"{checkpoints_namespace}/{model_last_name}"].upload(checkpoint_callback.last_model_path)
|
||||
|
||||
# save best k models
|
||||
if hasattr(checkpoint_callback, "best_k_models"):
|
||||
for key in checkpoint_callback.best_k_models:
|
||||
model_name = self._get_full_model_name(key, checkpoint_callback)
|
||||
file_names.add(model_name)
|
||||
self.run[f"{checkpoints_namespace}/{model_name}"].upload(key)
|
||||
|
||||
# log best model path and checkpoint
|
||||
if hasattr(checkpoint_callback, "best_model_path") and checkpoint_callback.best_model_path:
|
||||
self.run[self._construct_path_with_prefix("model/best_model_path")] = checkpoint_callback.best_model_path
|
||||
|
||||
model_name = self._get_full_model_name(checkpoint_callback.best_model_path, checkpoint_callback)
|
||||
file_names.add(model_name)
|
||||
self.run[f"{checkpoints_namespace}/{model_name}"].upload(checkpoint_callback.best_model_path)
|
||||
|
||||
# remove old models logged to experiment if they are not part of best k models at this point
|
||||
if self.run.exists(checkpoints_namespace):
|
||||
exp_structure = self.run.get_structure()
|
||||
uploaded_model_names = self._get_full_model_names_from_exp_structure(exp_structure, checkpoints_namespace)
|
||||
|
||||
for file_to_drop in list(uploaded_model_names - file_names):
|
||||
del self.run[f"{checkpoints_namespace}/{file_to_drop}"]
|
||||
|
||||
# log best model score
|
||||
if hasattr(checkpoint_callback, "best_model_score") and checkpoint_callback.best_model_score:
|
||||
self.run[self._construct_path_with_prefix("model/best_model_score")] = (
|
||||
checkpoint_callback.best_model_score.cpu().detach().numpy()
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _get_full_model_name(model_path: str, checkpoint_callback: Checkpoint) -> str:
|
||||
"""Returns model name which is string `model_path` appended to `checkpoint_callback.dirpath`."""
|
||||
if hasattr(checkpoint_callback, "dirpath"):
|
||||
model_path = os.path.normpath(model_path)
|
||||
expected_model_path = os.path.normpath(checkpoint_callback.dirpath)
|
||||
if not model_path.startswith(expected_model_path):
|
||||
raise ValueError(f"{model_path} was expected to start with {expected_model_path}.")
|
||||
# Remove extension from filepath
|
||||
filepath, _ = os.path.splitext(model_path[len(expected_model_path) + 1 :])
|
||||
return filepath.replace(os.sep, "/")
|
||||
return model_path.replace(os.sep, "/")
|
||||
|
||||
@classmethod
|
||||
def _get_full_model_names_from_exp_structure(cls, exp_structure: dict[str, Any], namespace: str) -> set[str]:
|
||||
"""Returns all paths to properties which were already logged in `namespace`"""
|
||||
structure_keys: list[str] = namespace.split(cls.LOGGER_JOIN_CHAR)
|
||||
for key in structure_keys:
|
||||
exp_structure = exp_structure[key]
|
||||
uploaded_models_dict = exp_structure
|
||||
return set(cls._dict_paths(uploaded_models_dict))
|
||||
|
||||
@classmethod
|
||||
def _dict_paths(cls, d: dict[str, Any], path_in_build: Optional[str] = None) -> Generator:
|
||||
for k, v in d.items():
|
||||
path = f"{path_in_build}/{k}" if path_in_build is not None else k
|
||||
if not isinstance(v, dict):
|
||||
yield path
|
||||
else:
|
||||
yield from cls._dict_paths(v, path)
|
||||
|
||||
@property
|
||||
@override
|
||||
def name(self) -> Optional[str]:
|
||||
"""Return the experiment name or 'offline-name' when exp is run in offline mode."""
|
||||
return self._run_name
|
||||
|
||||
@property
|
||||
@override
|
||||
def version(self) -> Optional[str]:
|
||||
"""Return the experiment version.
|
||||
|
||||
It's Neptune Run's short_id
|
||||
|
||||
"""
|
||||
return self._run_short_id
|
||||
|
||||
@@ -251,9 +251,6 @@ trainer = Trainer(logger=loggers.CometLogger())
|
||||
# mlflow
|
||||
trainer = Trainer(logger=loggers.MLFlowLogger())
|
||||
|
||||
# neptune
|
||||
trainer = Trainer(logger=loggers.NeptuneLogger())
|
||||
|
||||
# ... and dozens more
|
||||
```
|
||||
|
||||
|
||||
@@ -113,50 +113,6 @@ def comet_mock(monkeypatch):
|
||||
return comet
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def neptune_mock(monkeypatch):
|
||||
class RunType: # to make isinstance checks pass
|
||||
def get_root_object(self):
|
||||
pass
|
||||
|
||||
def __getitem__(self, item):
|
||||
pass
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
pass
|
||||
|
||||
run_mock = MagicMock(spec=RunType, exists=Mock(return_value=False), wait=Mock(), get_structure=MagicMock())
|
||||
run_mock.get_root_object.return_value = run_mock
|
||||
|
||||
neptune = ModuleType("neptune")
|
||||
neptune.init_run = Mock(return_value=run_mock)
|
||||
neptune.Run = RunType
|
||||
monkeypatch.setitem(sys.modules, "neptune", neptune)
|
||||
|
||||
neptune_handler = ModuleType("handler")
|
||||
neptune_handler.Handler = RunType
|
||||
monkeypatch.setitem(sys.modules, "neptune.handler", neptune_handler)
|
||||
|
||||
neptune_types = ModuleType("types")
|
||||
neptune_types.File = Mock()
|
||||
monkeypatch.setitem(sys.modules, "neptune.types", neptune_types)
|
||||
|
||||
neptune_utils = ModuleType("utils")
|
||||
neptune_utils.stringify_unsupported = Mock()
|
||||
monkeypatch.setitem(sys.modules, "neptune.utils", neptune_utils)
|
||||
|
||||
neptune_exceptions = ModuleType("exceptions")
|
||||
neptune_exceptions.InactiveRunException = Exception
|
||||
monkeypatch.setitem(sys.modules, "neptune.exceptions", neptune_exceptions)
|
||||
|
||||
neptune.handler = neptune_handler
|
||||
neptune.types = neptune_types
|
||||
neptune.utils = neptune_utils
|
||||
|
||||
monkeypatch.setattr("lightning.pytorch.loggers.neptune._NEPTUNE_AVAILABLE", True)
|
||||
return neptune
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def litlogger_mock(monkeypatch):
|
||||
"""Mock litlogger module for unit testing LightningLogger."""
|
||||
|
||||
@@ -26,7 +26,6 @@ from lightning.pytorch.loggers import (
|
||||
CometLogger,
|
||||
CSVLogger,
|
||||
MLFlowLogger,
|
||||
NeptuneLogger,
|
||||
TensorBoardLogger,
|
||||
WandbLogger,
|
||||
)
|
||||
@@ -41,11 +40,9 @@ ALL_LOGGER_CLASSES = (
|
||||
CometLogger,
|
||||
CSVLogger,
|
||||
MLFlowLogger,
|
||||
NeptuneLogger,
|
||||
TensorBoardLogger,
|
||||
WandbLogger,
|
||||
)
|
||||
ALL_LOGGER_CLASSES_WO_NEPTUNE = tuple(filter(lambda cls: cls is not NeptuneLogger, ALL_LOGGER_CLASSES))
|
||||
|
||||
|
||||
def _get_logger_args(logger_class, save_dir):
|
||||
@@ -56,8 +53,6 @@ def _get_logger_args(logger_class, save_dir):
|
||||
logger_args.update(offline_mode=True)
|
||||
if "offline" in inspect.getfullargspec(logger_class).args:
|
||||
logger_args.update(offline=True)
|
||||
if issubclass(logger_class, NeptuneLogger):
|
||||
logger_args.update(mode="offline")
|
||||
if issubclass(logger_class, CometLogger):
|
||||
logger_args.update(online=False)
|
||||
return logger_args
|
||||
@@ -72,7 +67,7 @@ def _instantiate_logger(logger_class, save_dir, **override_kwargs):
|
||||
@mock.patch.dict(os.environ, {})
|
||||
@mock.patch("lightning.pytorch.loggers.mlflow._get_resolve_tags", Mock())
|
||||
@pytest.mark.parametrize("logger_class", ALL_LOGGER_CLASSES)
|
||||
def test_loggers_fit_test_all(logger_class, mlflow_mock, wandb_mock, comet_mock, neptune_mock, tmp_path, monkeypatch):
|
||||
def test_loggers_fit_test_all(logger_class, mlflow_mock, wandb_mock, comet_mock, tmp_path, monkeypatch):
|
||||
"""Verify that basic functionality of all loggers."""
|
||||
monkeypatch.chdir(tmp_path)
|
||||
|
||||
@@ -111,11 +106,6 @@ def test_loggers_fit_test_all(logger_class, mlflow_mock, wandb_mock, comet_mock,
|
||||
logger._project_name = "bar"
|
||||
logger.experiment.get_key.return_value = "SOME_KEY"
|
||||
|
||||
if logger_class == NeptuneLogger:
|
||||
logger._retrieve_run_data = Mock()
|
||||
logger._run_short_id = "foo"
|
||||
logger._run_name = "bar"
|
||||
|
||||
if logger_class == MLFlowLogger:
|
||||
logger = mock_mlflow_run_creation(logger, experiment_id="foo", run_id="bar")
|
||||
|
||||
@@ -149,9 +139,7 @@ def test_loggers_fit_test_all(logger_class, mlflow_mock, wandb_mock, comet_mock,
|
||||
|
||||
|
||||
@mock.patch.dict(os.environ, {})
|
||||
@pytest.mark.parametrize(
|
||||
"logger_class", ALL_LOGGER_CLASSES_WO_NEPTUNE
|
||||
) # WandbLogger and NeptuneLogger get tested separately
|
||||
@pytest.mark.parametrize("logger_class", ALL_LOGGER_CLASSES) # WandbLogger gets tested separately
|
||||
def test_loggers_pickle_all(tmp_path, monkeypatch, logger_class):
|
||||
"""Test that the logger objects can be pickled.
|
||||
|
||||
@@ -222,8 +210,6 @@ class LazyInitExperimentCheck(Callback):
|
||||
return
|
||||
if isinstance(trainer.logger, MLFlowLogger):
|
||||
assert trainer.logger._mlflow_client
|
||||
elif isinstance(trainer.logger, NeptuneLogger):
|
||||
assert trainer.logger._run_instance
|
||||
elif hasattr(trainer.logger, "_experiment"):
|
||||
assert trainer.logger._experiment
|
||||
|
||||
@@ -255,7 +241,7 @@ class CustomLoggerWithoutExperiment(Logger):
|
||||
|
||||
|
||||
@mock.patch.dict(os.environ, {})
|
||||
@pytest.mark.parametrize("logger_class", [*ALL_LOGGER_CLASSES_WO_NEPTUNE, CustomLoggerWithoutExperiment])
|
||||
@pytest.mark.parametrize("logger_class", [*ALL_LOGGER_CLASSES, CustomLoggerWithoutExperiment])
|
||||
@RunIf(skip_windows=True)
|
||||
def test_logger_initialization(tmp_path, monkeypatch, logger_class):
|
||||
"""Test that loggers get replaced by dummy loggers on global rank > 0 and that the experiment object is available
|
||||
@@ -288,7 +274,7 @@ def _test_logger_initialization(tmp_path, logger_class):
|
||||
|
||||
@mock.patch.dict(os.environ, {})
|
||||
@mock.patch("lightning.pytorch.loggers.mlflow._get_resolve_tags", Mock())
|
||||
def test_logger_with_prefix_all(mlflow_mock, wandb_mock, comet_mock, neptune_mock, monkeypatch, tmp_path):
|
||||
def test_logger_with_prefix_all(mlflow_mock, wandb_mock, comet_mock, monkeypatch, tmp_path):
|
||||
"""Test that prefix is added at the beginning of the metric keys."""
|
||||
prefix = "tmp"
|
||||
|
||||
@@ -308,14 +294,6 @@ def test_logger_with_prefix_all(mlflow_mock, wandb_mock, comet_mock, neptune_moc
|
||||
run_id=ANY, metrics=[Metric(key="tmp-test", value=1.0, timestamp=ANY, step=0)]
|
||||
)
|
||||
|
||||
# Neptune
|
||||
logger = _instantiate_logger(NeptuneLogger, api_key="test", project="project", save_dir=tmp_path, prefix=prefix)
|
||||
assert logger.experiment.__getitem__.call_count == 0
|
||||
logger.log_metrics({"test": 1.0}, step=0)
|
||||
assert logger.experiment.__getitem__.call_count == 1
|
||||
logger.experiment.__getitem__.assert_called_with("tmp/test")
|
||||
logger.experiment.__getitem__().append.assert_called_once_with(1.0, step=0)
|
||||
|
||||
# TensorBoard
|
||||
if _TENSORBOARD_AVAILABLE:
|
||||
import torch.utils.tensorboard as tb
|
||||
|
||||
@@ -21,6 +21,7 @@ import torch
|
||||
from lightning.pytorch import Trainer
|
||||
from lightning.pytorch.demos.boring_classes import BoringModel
|
||||
from lightning.pytorch.loggers.litlogger import LitLogger
|
||||
from lightning.pytorch.loggers.neptune import NeptuneLogger
|
||||
|
||||
|
||||
def test_litlogger_init(litlogger_mock, tmp_path):
|
||||
@@ -360,3 +361,8 @@ def test_litlogger_save_logs_option(litlogger_mock, tmp_path):
|
||||
|
||||
call_kwargs = litlogger_mock.Experiment.call_args[1]
|
||||
assert call_kwargs["save_logs"] is True
|
||||
|
||||
|
||||
def test_neptune_logger_raises_runtime_error():
|
||||
with pytest.raises(RuntimeError, match="NeptuneLogger is no longer supported"):
|
||||
NeptuneLogger()
|
||||
|
||||
@@ -1,318 +0,0 @@
|
||||
# Copyright The Lightning AI team.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import os
|
||||
import pickle
|
||||
from collections import namedtuple
|
||||
from unittest import mock
|
||||
from unittest.mock import MagicMock, call
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import lightning.pytorch as pl
|
||||
from lightning.pytorch import Trainer
|
||||
from lightning.pytorch.demos.boring_classes import BoringModel
|
||||
from lightning.pytorch.loggers import NeptuneLogger
|
||||
|
||||
|
||||
def _fetchable_paths(value):
|
||||
if value == "sys/id":
|
||||
return MagicMock(fetch=MagicMock(return_value="TEST-1"))
|
||||
if value == "sys/name":
|
||||
return MagicMock(fetch=MagicMock(return_value="Run test name"))
|
||||
return MagicMock()
|
||||
|
||||
|
||||
def _fit_and_test(logger, model, tmp_path):
|
||||
trainer = Trainer(default_root_dir=tmp_path, max_epochs=1, limit_train_batches=0.05, logger=logger)
|
||||
assert trainer.log_dir == os.path.join(os.getcwd(), ".neptune")
|
||||
trainer.fit(model)
|
||||
trainer.test(model)
|
||||
assert trainer.log_dir == os.path.join(os.getcwd(), ".neptune")
|
||||
|
||||
|
||||
def _get_logger_with_mocks(**kwargs):
|
||||
logger = NeptuneLogger(**kwargs)
|
||||
run_instance_mock = MagicMock()
|
||||
logger._run_instance = run_instance_mock
|
||||
logger._run_instance.__getitem__.return_value.fetch.return_value = "exp-name"
|
||||
run_attr_mock = MagicMock()
|
||||
logger._run_instance.__getitem__.return_value = run_attr_mock
|
||||
|
||||
return logger, run_instance_mock, run_attr_mock
|
||||
|
||||
|
||||
def test_neptune_online(neptune_mock):
|
||||
neptune_mock.init_run.return_value.exists.return_value = True
|
||||
neptune_mock.init_run.return_value.__getitem__.side_effect = _fetchable_paths
|
||||
|
||||
logger = NeptuneLogger(api_key="test", project="project")
|
||||
created_run_mock = logger.run
|
||||
|
||||
assert logger._run_instance == created_run_mock
|
||||
created_run_mock.exists.assert_called_once_with("sys/id")
|
||||
assert logger.name == "Run test name"
|
||||
assert logger.version == "TEST-1"
|
||||
assert neptune_mock.init_run.call_count == 1
|
||||
assert created_run_mock.__getitem__.call_count == 2
|
||||
assert created_run_mock.__setitem__.call_count == 1
|
||||
created_run_mock.__getitem__.assert_has_calls([call("sys/id"), call("sys/name")], any_order=True)
|
||||
created_run_mock.__setitem__.assert_called_once_with("source_code/integrations/pytorch-lightning", pl.__version__)
|
||||
|
||||
|
||||
def test_neptune_offline(neptune_mock):
|
||||
neptune_mock.init_run.return_value.exists.return_value = False
|
||||
|
||||
logger = NeptuneLogger(mode="offline")
|
||||
created_run_mock = logger.run
|
||||
logger.experiment["foo"] = "bar"
|
||||
|
||||
created_run_mock.exists.assert_called_once_with("sys/id")
|
||||
assert logger._run_short_id == "OFFLINE"
|
||||
assert logger._run_name == "offline-name"
|
||||
|
||||
|
||||
def test_online_with_custom_run(neptune_mock):
|
||||
neptune_mock.init_run.return_value.exists.return_value = True
|
||||
neptune_mock.init_run.return_value.__getitem__.side_effect = _fetchable_paths
|
||||
|
||||
created_run = neptune_mock.init_run()
|
||||
neptune_mock.init_run.reset_mock()
|
||||
|
||||
logger = NeptuneLogger(run=created_run)
|
||||
assert logger._run_instance == created_run
|
||||
assert logger._run_instance == created_run
|
||||
assert logger.version == "TEST-1"
|
||||
assert neptune_mock.init_run.call_count == 0
|
||||
|
||||
|
||||
def test_neptune_pickling(neptune_mock):
|
||||
neptune_mock.init_run.return_value.exists.return_value = True
|
||||
neptune_mock.init_run.return_value.__getitem__.side_effect = _fetchable_paths
|
||||
|
||||
unpickleable_run = neptune_mock.init_run()
|
||||
with pytest.raises(pickle.PicklingError):
|
||||
pickle.dumps(unpickleable_run)
|
||||
neptune_mock.init_run.reset_mock()
|
||||
|
||||
logger = NeptuneLogger(run=unpickleable_run)
|
||||
assert neptune_mock.init_run.call_count == 0
|
||||
|
||||
pickled_logger = pickle.dumps(logger)
|
||||
unpickled = pickle.loads(pickled_logger)
|
||||
|
||||
neptune_mock.init_run.assert_called_once_with(name="Run test name", run="TEST-1")
|
||||
assert unpickled.experiment is not None
|
||||
|
||||
|
||||
def test_online_with_wrong_kwargs(neptune_mock):
|
||||
"""Tests combinations of kwargs together with `run` kwarg which makes some of other parameters unavailable in
|
||||
init."""
|
||||
run = neptune_mock.init_run()
|
||||
|
||||
with pytest.raises(ValueError, match="Run parameter expected to be of type `neptune.Run`*"):
|
||||
NeptuneLogger(run="some string")
|
||||
|
||||
with pytest.raises(ValueError, match="When an already initialized run object is provided*"):
|
||||
NeptuneLogger(run=run, project="redundant project")
|
||||
|
||||
with pytest.raises(ValueError, match="When an already initialized run object is provided*"):
|
||||
NeptuneLogger(run=run, api_key="redundant api key")
|
||||
|
||||
with pytest.raises(ValueError, match="When an already initialized run object is provided*"):
|
||||
NeptuneLogger(run=run, name="redundant api name")
|
||||
|
||||
with pytest.raises(ValueError, match="When an already initialized run object is provided*"):
|
||||
NeptuneLogger(run=run, foo="random **kwarg")
|
||||
|
||||
# this should work
|
||||
NeptuneLogger(run=run)
|
||||
NeptuneLogger(project="foo")
|
||||
NeptuneLogger(foo="bar")
|
||||
|
||||
|
||||
def test_neptune_additional_methods(neptune_mock):
|
||||
logger, run_instance_mock, _ = _get_logger_with_mocks(api_key="test", project="project")
|
||||
|
||||
logger.experiment["key1"].log(torch.ones(1))
|
||||
run_instance_mock.__getitem__.assert_called_once_with("key1")
|
||||
run_instance_mock.__getitem__().log.assert_called_once_with(torch.ones(1))
|
||||
|
||||
|
||||
def test_neptune_leave_open_experiment_after_fit(neptune_mock, tmp_path, monkeypatch):
|
||||
"""Verify that neptune experiment was NOT closed after training."""
|
||||
monkeypatch.chdir(tmp_path)
|
||||
logger, run_instance_mock, _ = _get_logger_with_mocks(api_key="test", project="project")
|
||||
_fit_and_test(logger=logger, model=BoringModel(), tmp_path=tmp_path)
|
||||
assert run_instance_mock.stop.call_count == 0
|
||||
|
||||
|
||||
def test_neptune_log_metrics_on_trained_model(neptune_mock, tmp_path, monkeypatch):
|
||||
"""Verify that trained models do log data."""
|
||||
monkeypatch.chdir(tmp_path)
|
||||
|
||||
class LoggingModel(BoringModel):
|
||||
def on_validation_epoch_end(self):
|
||||
self.log("some/key", 42)
|
||||
|
||||
logger, run_instance_mock, _ = _get_logger_with_mocks(api_key="test", project="project")
|
||||
_fit_and_test(logger=logger, model=LoggingModel(), tmp_path=tmp_path)
|
||||
run_instance_mock.__getitem__.assert_any_call("training/some/key")
|
||||
run_instance_mock.__getitem__.return_value.append.assert_has_calls([call(42, step=2)])
|
||||
|
||||
|
||||
def test_log_hyperparams(neptune_mock):
|
||||
neptune_mock.utils.stringify_unsupported = lambda x: x
|
||||
|
||||
params = {"foo": "bar", "nested_foo": {"bar": 42}}
|
||||
test_variants = [
|
||||
({}, "training/hyperparams"),
|
||||
({"prefix": "custom_prefix"}, "custom_prefix/hyperparams"),
|
||||
({"prefix": "custom/nested/prefix"}, "custom/nested/prefix/hyperparams"),
|
||||
]
|
||||
for prefix, hyperparams_key in test_variants:
|
||||
logger, run_instance_mock, _ = _get_logger_with_mocks(api_key="test", project="project", **prefix)
|
||||
logger.log_hyperparams(params)
|
||||
assert run_instance_mock.__setitem__.call_count == 1
|
||||
assert run_instance_mock.__getitem__.call_count == 0
|
||||
run_instance_mock.__setitem__.assert_called_once_with(hyperparams_key, params)
|
||||
|
||||
|
||||
def test_log_metrics(neptune_mock):
|
||||
metrics = {
|
||||
"foo": 42,
|
||||
"bar": 555,
|
||||
}
|
||||
test_variants = [
|
||||
({}, ("training/foo", "training/bar")),
|
||||
({"prefix": "custom_prefix"}, ("custom_prefix/foo", "custom_prefix/bar")),
|
||||
({"prefix": "custom/nested/prefix"}, ("custom/nested/prefix/foo", "custom/nested/prefix/bar")),
|
||||
]
|
||||
|
||||
for prefix, (metrics_foo_key, metrics_bar_key) in test_variants:
|
||||
logger, run_instance_mock, run_attr_mock = _get_logger_with_mocks(api_key="test", project="project", **prefix)
|
||||
logger.log_metrics(metrics)
|
||||
assert run_instance_mock.__setitem__.call_count == 0
|
||||
assert run_instance_mock.__getitem__.call_count == 2
|
||||
run_instance_mock.__getitem__.assert_any_call(metrics_foo_key)
|
||||
run_instance_mock.__getitem__.assert_any_call(metrics_bar_key)
|
||||
run_attr_mock.append.assert_has_calls([call(42, step=None), call(555, step=None)])
|
||||
|
||||
|
||||
def test_log_model_summary(neptune_mock):
|
||||
model = BoringModel()
|
||||
test_variants = [
|
||||
({}, "training/model/summary"),
|
||||
({"prefix": "custom_prefix"}, "custom_prefix/model/summary"),
|
||||
({"prefix": "custom/nested/prefix"}, "custom/nested/prefix/model/summary"),
|
||||
]
|
||||
|
||||
for prefix, model_summary_key in test_variants:
|
||||
logger, run_instance_mock, _ = _get_logger_with_mocks(api_key="test", project="project", **prefix)
|
||||
file_from_content_mock = neptune_mock.types.File.from_content()
|
||||
|
||||
logger.log_model_summary(model)
|
||||
|
||||
assert run_instance_mock.__setitem__.call_count == 1
|
||||
assert run_instance_mock.__getitem__.call_count == 0
|
||||
run_instance_mock.__setitem__.assert_called_once_with(model_summary_key, file_from_content_mock)
|
||||
|
||||
|
||||
@mock.patch("builtins.open", mock.mock_open(read_data="test"))
|
||||
def test_after_save_checkpoint(neptune_mock):
|
||||
test_variants = [
|
||||
({}, "training/model"),
|
||||
({"prefix": "custom_prefix"}, "custom_prefix/model"),
|
||||
({"prefix": "custom/nested/prefix"}, "custom/nested/prefix/model"),
|
||||
]
|
||||
|
||||
for prefix, model_key_prefix in test_variants:
|
||||
logger, run_instance_mock, run_attr_mock = _get_logger_with_mocks(api_key="test", project="project", **prefix)
|
||||
models_root_dir = os.path.join("path", "to", "models")
|
||||
cb_mock = MagicMock(
|
||||
dirpath=models_root_dir,
|
||||
last_model_path=os.path.join(models_root_dir, "last"),
|
||||
best_k_models={
|
||||
f"{os.path.join(models_root_dir, 'model1')}": None,
|
||||
f"{os.path.join(models_root_dir, 'model2/with/slashes')}": None,
|
||||
},
|
||||
best_model_path=os.path.join(models_root_dir, "best_model"),
|
||||
best_model_score=None,
|
||||
)
|
||||
|
||||
mock_file = neptune_mock.types.File
|
||||
mock_file.reset_mock()
|
||||
mock_file.side_effect = mock.Mock()
|
||||
logger.after_save_checkpoint(cb_mock)
|
||||
|
||||
assert run_instance_mock.__setitem__.call_count == 1 # best_model_path
|
||||
assert run_instance_mock.__getitem__.call_count == 4 # last_model_path, best_k_models, best_model_path
|
||||
assert run_attr_mock.upload.call_count == 4 # last_model_path, best_k_models, best_model_path
|
||||
assert mock_file.from_stream.call_count == 0
|
||||
|
||||
run_instance_mock.__getitem__.assert_any_call(f"{model_key_prefix}/checkpoints/model1")
|
||||
run_instance_mock.__getitem__.assert_any_call(f"{model_key_prefix}/checkpoints/model2/with/slashes")
|
||||
|
||||
run_attr_mock.upload.assert_has_calls([
|
||||
call(os.path.join(models_root_dir, "model1")),
|
||||
call(os.path.join(models_root_dir, "model2/with/slashes")),
|
||||
])
|
||||
|
||||
|
||||
def test_save_dir(neptune_mock):
|
||||
logger = NeptuneLogger(api_key="test", project="project")
|
||||
assert logger.save_dir == os.path.join(os.getcwd(), ".neptune")
|
||||
|
||||
|
||||
def test_get_full_model_name():
|
||||
SimpleCheckpoint = namedtuple("SimpleCheckpoint", ["dirpath"])
|
||||
test_input_data = [
|
||||
("key", os.path.join("foo", "bar", "key.ext"), SimpleCheckpoint(dirpath=os.path.join("foo", "bar"))),
|
||||
(
|
||||
"key/in/parts",
|
||||
os.path.join("foo", "bar", "key/in/parts.ext"),
|
||||
SimpleCheckpoint(dirpath=os.path.join("foo", "bar")),
|
||||
),
|
||||
("key", os.path.join("../foo", "bar", "key.ext"), SimpleCheckpoint(dirpath=os.path.join("../foo", "bar"))),
|
||||
("key", os.path.join("foo", "key.ext"), SimpleCheckpoint(dirpath=os.path.join("./foo", "bar/../"))),
|
||||
]
|
||||
|
||||
for expected_model_name, model_path, checkpoint in test_input_data:
|
||||
assert NeptuneLogger._get_full_model_name(model_path, checkpoint) == expected_model_name
|
||||
|
||||
|
||||
def test_get_full_model_names_from_exp_structure():
|
||||
input_dict = {
|
||||
"foo": {
|
||||
"bar": {
|
||||
"lvl1_1": {"lvl2": {"lvl3_1": "some non important value", "lvl3_2": "some non important value"}},
|
||||
"lvl1_2": "some non important value",
|
||||
},
|
||||
"other_non_important": {"val100": 100},
|
||||
},
|
||||
"other_non_important": {"val42": 42},
|
||||
}
|
||||
expected_keys = {"lvl1_1/lvl2/lvl3_1", "lvl1_1/lvl2/lvl3_2", "lvl1_2"}
|
||||
assert NeptuneLogger._get_full_model_names_from_exp_structure(input_dict, "foo/bar") == expected_keys
|
||||
|
||||
|
||||
def test_inactive_run(neptune_mock, tmp_path, monkeypatch):
|
||||
from neptune.exceptions import InactiveRunException
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
logger, run_instance_mock, _ = _get_logger_with_mocks(api_key="test", project="project")
|
||||
run_instance_mock.__setitem__.side_effect = InactiveRunException
|
||||
|
||||
# this should work without any exceptions
|
||||
_fit_and_test(logger=logger, model=BoringModel(), tmp_path=tmp_path)
|
||||
@@ -1802,21 +1802,6 @@ def test_comet_logger_init_args():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.xfail(
|
||||
# Only on Windows: TypeError: 'NoneType' object is not subscriptable
|
||||
raises=TypeError,
|
||||
condition=(sys.platform == "win32"),
|
||||
strict=False,
|
||||
reason="TypeError on Windows when parsing",
|
||||
)
|
||||
def test_neptune_logger_init_args():
|
||||
_test_logger_init_args(
|
||||
"NeptuneLogger",
|
||||
init={"name": "neptune"}, # Resolve from NeptuneLogger.__init__
|
||||
unresolved={"description": "neptune"}, # Unsupported resolving from neptune.internal.init.run.init_run
|
||||
)
|
||||
|
||||
|
||||
def test_tensorboard_logger_init_args():
|
||||
_test_logger_init_args(
|
||||
"TensorBoardLogger",
|
||||
|
||||
@@ -85,12 +85,6 @@ def clean_import():
|
||||
pytest.param(
|
||||
"torch.distributed.is_available", _shortcut_patch(is_available, ()), "lightning.pytorch", id="ProcessGroup"
|
||||
),
|
||||
pytest.param(
|
||||
"lightning_utilities.core.imports.RequirementCache.__bool__",
|
||||
_shortcut_patch(RequirementCache.__bool__, ("neptune",), ("requirement",)),
|
||||
"lightning.pytorch.loggers.neptune",
|
||||
id="neptune",
|
||||
),
|
||||
pytest.param(
|
||||
"lightning_utilities.core.imports.RequirementCache.__bool__",
|
||||
_shortcut_patch(RequirementCache.__bool__, ("jsonargparse[signatures]>=4.12.0",), ("requirement",)),
|
||||
|
||||
Reference in New Issue
Block a user