From c2a5fe62afaa0629514ba1d7a94003c8f0ef80a3 Mon Sep 17 00:00:00 2001 From: HydrogenSulfate <490868991@qq.com> Date: Thu, 21 Dec 2023 01:57:39 +0800 Subject: [PATCH] simplify code and remove train_dataloader_cfg in most of case --- .pre-commit-config.yaml | 2 +- docs/zh/examples/ldc2d_steady.md | 2 +- docs/zh/examples/ldc2d_unsteady.md | 2 +- examples/darcy/darcy2d.py | 16 ++-------------- examples/gpinn/poisson_1d.py | 12 +----------- examples/heat_pinn/heat_pinn.py | 16 +++++----------- examples/laplace/laplace2d.py | 10 ++-------- examples/ldc/ldc2d_steady_Re10.py | 20 +++++--------------- examples/ldc/ldc2d_unsteady_Re10.py | 22 ++++++---------------- ppsci/constraint/boundary_constraint.py | 5 +++++ ppsci/constraint/initial_constraint.py | 5 +++++ ppsci/constraint/integral_constraint.py | 5 +++++ ppsci/constraint/interior_constraint.py | 5 +++++ ppsci/constraint/periodic_constraint.py | 5 +++++ ppsci/constraint/supervised_constraint.py | 5 +++++ ppsci/data/__init__.py | 22 +++++++++++----------- ppsci/data/dataset/__init__.py | 7 +++++-- ppsci/validate/geo_validator.py | 5 +++++ ppsci/validate/sup_validator.py | 5 +++++ 19 files changed, 80 insertions(+), 91 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index aa6f84d71..8132db56b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: - id: black - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: 'v0.0.272' + rev: "v0.0.272" hooks: - id: ruff diff --git a/docs/zh/examples/ldc2d_steady.md b/docs/zh/examples/ldc2d_steady.md index 037dc50bc..43b9dfe80 100644 --- a/docs/zh/examples/ldc2d_steady.md +++ b/docs/zh/examples/ldc2d_steady.md @@ -221,7 +221,7 @@ pde = ppsci.constraint.InteriorConstraint( equation["NavierStokes"].equations, {"continuity": 0, "momentum_x": 0, "momentum_y": 0}, geom["rect"], - {**train_dataloader_cfg, "batch_size": NPOINT_PDE}, + {"batch_size": NPOINT_PDE}, ppsci.loss.MSELoss("sum"), evenly=True, weight_dict=cfg.TRAIN.weight.pde, # (1) diff --git a/docs/zh/examples/ldc2d_unsteady.md b/docs/zh/examples/ldc2d_unsteady.md index 869836229..228f36740 100644 --- a/docs/zh/examples/ldc2d_unsteady.md +++ b/docs/zh/examples/ldc2d_unsteady.md @@ -231,7 +231,7 @@ pde = ppsci.constraint.InteriorConstraint( equation["NavierStokes"].equations, {"continuity": 0, "momentum_x": 0, "momentum_y": 0}, geom["time_rect"], - {**train_dataloader_cfg, "batch_size": NPOINT_PDE * NTIME_PDE}, + {"batch_size": NPOINT_PDE * NTIME_PDE}, ppsci.loss.MSELoss("sum"), evenly=True, weight_dict=cfg.TRAIN.weight.pde, # (1) diff --git a/examples/darcy/darcy2d.py b/examples/darcy/darcy2d.py index 7f0a3646f..8709e12de 100644 --- a/examples/darcy/darcy2d.py +++ b/examples/darcy/darcy2d.py @@ -39,12 +39,6 @@ def train(cfg: DictConfig): # set geometry geom = {"rect": ppsci.geometry.Rectangle((0.0, 0.0), (1.0, 1.0))} - # set dataloader config - train_dataloader_cfg = { - "dataset": "IterableNamedArrayDataset", - "iters_per_epoch": cfg.TRAIN.iters_per_epoch, - } - # set constraint def poisson_ref_compute_func(_in): return ( @@ -58,7 +52,7 @@ def poisson_ref_compute_func(_in): equation["Poisson"].equations, {"poisson": poisson_ref_compute_func}, geom["rect"], - {**train_dataloader_cfg, "batch_size": cfg.NPOINT_PDE}, + {"batch_size": cfg.NPOINT_PDE}, ppsci.loss.MSELoss("sum"), evenly=True, name="EQ", @@ -71,7 +65,7 @@ def poisson_ref_compute_func(_in): * np.cos(2.0 * np.pi * _in["y"]) }, geom["rect"], - {**train_dataloader_cfg, "batch_size": cfg.NPOINT_BC}, + {"batch_size": cfg.NPOINT_BC}, ppsci.loss.MSELoss("sum"), name="BC", ) @@ -91,10 +85,8 @@ def poisson_ref_compute_func(_in): {"poisson": poisson_ref_compute_func}, geom["rect"], { - "dataset": "NamedArrayDataset", "total_size": cfg.NPOINT_PDE, "batch_size": cfg.EVAL.batch_size.residual_validator, - "sampler": {"name": "BatchSampler"}, }, ppsci.loss.MSELoss("sum"), evenly=True, @@ -157,7 +149,6 @@ def poisson_ref_compute_func(_in): eval_during_train=cfg.TRAIN.eval_during_train, eval_freq=cfg.TRAIN.eval_freq, equation=equation, - geom=geom, validator=validator, visualizer=visualizer, ) @@ -186,7 +177,6 @@ def poisson_ref_compute_func(_in): eval_during_train=cfg.TRAIN.lbfgs.eval_during_train, eval_freq=cfg.TRAIN.lbfgs.eval_freq, equation=equation, - geom=geom, validator=validator, visualizer=visualizer, ) @@ -228,10 +218,8 @@ def poisson_ref_compute_func(_in): {"poisson": poisson_ref_compute_func}, geom["rect"], { - "dataset": "NamedArrayDataset", "total_size": cfg.NPOINT_PDE, "batch_size": cfg.EVAL.batch_size.residual_validator, - "sampler": {"name": "BatchSampler"}, }, ppsci.loss.MSELoss("sum"), evenly=True, diff --git a/examples/gpinn/poisson_1d.py b/examples/gpinn/poisson_1d.py index a8cb3da1b..dc0e112b1 100644 --- a/examples/gpinn/poisson_1d.py +++ b/examples/gpinn/poisson_1d.py @@ -83,18 +83,12 @@ def output_transform( # set geometry geom = {"line": ppsci.geometry.Interval(0, np.pi)} - # set dataloader config - train_dataloader_cfg = { - "dataset": "IterableNamedArrayDataset", - "iters_per_epoch": cfg.TRAIN.iters_per_epoch, - } - # set constraint pde_constraint = ppsci.constraint.InteriorConstraint( equation["gPINN"].equations, {"res1": 0, "res2": 0}, geom["line"], - {**train_dataloader_cfg, "batch_size": cfg.NPOINT_PDE}, + {"batch_size": cfg.NPOINT_PDE}, ppsci.loss.MSELoss("mean", weight={"res2": 0.01}), evenly=True, name="EQ", @@ -120,10 +114,8 @@ def u_solution(in_): {outvar: u_solution}, geom["line"], { - "dataset": "NamedArrayDataset", "total_size": cfg.NPOINT_PDE_EVAL, "batch_size": cfg.EVAL.batch_size.l2rel_validator, - "sampler": {"name": "BatchSampler"}, }, ppsci.loss.MSELoss("mean"), evenly=True, @@ -250,10 +242,8 @@ def u_solution(in_): {outvar: u_solution}, geom["line"], { - "dataset": "NamedArrayDataset", "total_size": cfg.NPOINT_PDE, "batch_size": cfg.EVAL.batch_size.l2rel_validator, - "sampler": {"name": "BatchSampler"}, }, ppsci.loss.MSELoss("mean"), evenly=True, diff --git a/examples/heat_pinn/heat_pinn.py b/examples/heat_pinn/heat_pinn.py index 972095775..ebf781f88 100644 --- a/examples/heat_pinn/heat_pinn.py +++ b/examples/heat_pinn/heat_pinn.py @@ -40,12 +40,6 @@ def train(cfg: DictConfig): # set geometry geom = {"rect": ppsci.geometry.Rectangle((-1.0, -1.0), (1.0, 1.0))} - # set train dataloader config - train_dataloader_cfg = { - "dataset": "IterableNamedArrayDataset", - "iters_per_epoch": cfg.TRAIN.iters_per_epoch, - } - # set constraint NPOINT_PDE = 99**2 NPOINT_TOP = 25 @@ -56,7 +50,7 @@ def train(cfg: DictConfig): equation["heat"].equations, {"laplace": 0}, geom["rect"], - {**train_dataloader_cfg, "batch_size": NPOINT_PDE}, + {"batch_size": NPOINT_PDE}, ppsci.loss.MSELoss("mean"), evenly=True, name="EQ", @@ -65,7 +59,7 @@ def train(cfg: DictConfig): {"u": lambda out: out["u"]}, {"u": 0}, geom["rect"], - {**train_dataloader_cfg, "batch_size": NPOINT_TOP}, + {"batch_size": NPOINT_TOP}, ppsci.loss.MSELoss("mean"), weight_dict={"u": cfg.TRAIN.weight.bc_top}, criteria=lambda x, y: np.isclose(y, 1), @@ -75,7 +69,7 @@ def train(cfg: DictConfig): {"u": lambda out: out["u"]}, {"u": 50 / 75}, geom["rect"], - {**train_dataloader_cfg, "batch_size": NPOINT_BOTTOM}, + {"batch_size": NPOINT_BOTTOM}, ppsci.loss.MSELoss("mean"), weight_dict={"u": cfg.TRAIN.weight.bc_bottom}, criteria=lambda x, y: np.isclose(y, -1), @@ -85,7 +79,7 @@ def train(cfg: DictConfig): {"u": lambda out: out["u"]}, {"u": 1}, geom["rect"], - {**train_dataloader_cfg, "batch_size": NPOINT_LEFT}, + {"batch_size": NPOINT_LEFT}, ppsci.loss.MSELoss("mean"), weight_dict={"u": cfg.TRAIN.weight.bc_left}, criteria=lambda x, y: np.isclose(x, -1), @@ -95,7 +89,7 @@ def train(cfg: DictConfig): {"u": lambda out: out["u"]}, {"u": 0}, geom["rect"], - {**train_dataloader_cfg, "batch_size": NPOINT_RIGHT}, + {"batch_size": NPOINT_RIGHT}, ppsci.loss.MSELoss("mean"), weight_dict={"u": cfg.TRAIN.weight.bc_right}, criteria=lambda x, y: np.isclose(x, 1), diff --git a/examples/laplace/laplace2d.py b/examples/laplace/laplace2d.py index 603c63bf4..46f6a14d3 100644 --- a/examples/laplace/laplace2d.py +++ b/examples/laplace/laplace2d.py @@ -48,12 +48,6 @@ def u_solution_func(out): x, y = out["x"], out["y"] return np.cos(x) * np.cosh(y) - # set train dataloader config - train_dataloader_cfg = { - "dataset": "IterableNamedArrayDataset", - "iters_per_epoch": cfg.TRAIN.iters_per_epoch, - } - NPOINT_TOTAL = cfg.NPOINT_INTERIOR + cfg.NPOINT_BC # set constraint @@ -61,7 +55,7 @@ def u_solution_func(out): equation["laplace"].equations, {"laplace": 0}, geom["rect"], - {**train_dataloader_cfg, "batch_size": NPOINT_TOTAL}, + {"batch_size": NPOINT_TOTAL}, ppsci.loss.MSELoss("sum"), evenly=True, name="EQ", @@ -70,7 +64,7 @@ def u_solution_func(out): {"u": lambda out: out["u"]}, {"u": u_solution_func}, geom["rect"], - {**train_dataloader_cfg, "batch_size": cfg.NPOINT_BC}, + {"batch_size": cfg.NPOINT_BC}, ppsci.loss.MSELoss("sum"), name="BC", ) diff --git a/examples/ldc/ldc2d_steady_Re10.py b/examples/ldc/ldc2d_steady_Re10.py index c7eec4f3e..73cc42d08 100644 --- a/examples/ldc/ldc2d_steady_Re10.py +++ b/examples/ldc/ldc2d_steady_Re10.py @@ -36,12 +36,6 @@ def train(cfg: DictConfig): # set geometry geom = {"rect": ppsci.geometry.Rectangle((-0.05, -0.05), (0.05, 0.05))} - # set dataloader config - train_dataloader_cfg = { - "dataset": "IterableNamedArrayDataset", - "iters_per_epoch": cfg.TRAIN.iters_per_epoch, - } - NPOINT_PDE = 99**2 NPOINT_TOP = 101 NPOINT_BOTTOM = 101 @@ -53,7 +47,7 @@ def train(cfg: DictConfig): equation["NavierStokes"].equations, {"continuity": 0, "momentum_x": 0, "momentum_y": 0}, geom["rect"], - {**train_dataloader_cfg, "batch_size": NPOINT_PDE}, + {"batch_size": NPOINT_PDE}, ppsci.loss.MSELoss("sum"), evenly=True, weight_dict=cfg.TRAIN.weight.pde, @@ -63,7 +57,7 @@ def train(cfg: DictConfig): {"u": lambda out: out["u"], "v": lambda out: out["v"]}, {"u": 1, "v": 0}, geom["rect"], - {**train_dataloader_cfg, "batch_size": NPOINT_TOP}, + {"batch_size": NPOINT_TOP}, ppsci.loss.MSELoss("sum"), criteria=lambda x, y: np.isclose(y, 0.05), name="BC_top", @@ -72,7 +66,7 @@ def train(cfg: DictConfig): {"u": lambda out: out["u"], "v": lambda out: out["v"]}, {"u": 0, "v": 0}, geom["rect"], - {**train_dataloader_cfg, "batch_size": NPOINT_BOTTOM}, + {"batch_size": NPOINT_BOTTOM}, ppsci.loss.MSELoss("sum"), criteria=lambda x, y: np.isclose(y, -0.05), name="BC_bottom", @@ -81,7 +75,7 @@ def train(cfg: DictConfig): {"u": lambda out: out["u"], "v": lambda out: out["v"]}, {"u": 0, "v": 0}, geom["rect"], - {**train_dataloader_cfg, "batch_size": NPOINT_LEFT}, + {"batch_size": NPOINT_LEFT}, ppsci.loss.MSELoss("sum"), criteria=lambda x, y: np.isclose(x, -0.05), name="BC_left", @@ -90,7 +84,7 @@ def train(cfg: DictConfig): {"u": lambda out: out["u"], "v": lambda out: out["v"]}, {"u": 0, "v": 0}, geom["rect"], - {**train_dataloader_cfg, "batch_size": NPOINT_RIGHT}, + {"batch_size": NPOINT_RIGHT}, ppsci.loss.MSELoss("sum"), criteria=lambda x, y: np.isclose(x, 0.05), name="BC_right", @@ -118,10 +112,8 @@ def train(cfg: DictConfig): {"momentum_x": 0, "continuity": 0, "momentum_y": 0}, geom["rect"], { - "dataset": "NamedArrayDataset", "total_size": NPOINT_EVAL, "batch_size": cfg.EVAL.batch_size.residual_validator, - "sampler": {"name": "BatchSampler"}, }, ppsci.loss.MSELoss("sum"), evenly=True, @@ -195,10 +187,8 @@ def evaluate(cfg: DictConfig): {"momentum_x": 0, "continuity": 0, "momentum_y": 0}, geom["rect"], { - "dataset": "NamedArrayDataset", "total_size": NPOINT_EVAL, "batch_size": cfg.EVAL.batch_size.residual_validator, - "sampler": {"name": "BatchSampler"}, }, ppsci.loss.MSELoss("sum"), evenly=True, diff --git a/examples/ldc/ldc2d_unsteady_Re10.py b/examples/ldc/ldc2d_unsteady_Re10.py index 6d093e057..75030b3f8 100644 --- a/examples/ldc/ldc2d_unsteady_Re10.py +++ b/examples/ldc/ldc2d_unsteady_Re10.py @@ -43,12 +43,6 @@ def train(cfg: DictConfig): ) } - # set dataloader config - train_dataloader_cfg = { - "dataset": "IterableNamedArrayDataset", - "iters_per_epoch": cfg.TRAIN.iters_per_epoch, - } - # pde/bc constraint use t1~tn, initial constraint use t0 NPOINT_PDE, NTIME_PDE = 99**2, cfg.NTIME_ALL - 1 NPOINT_TOP, NTIME_TOP = 101, cfg.NTIME_ALL - 1 @@ -62,7 +56,7 @@ def train(cfg: DictConfig): equation["NavierStokes"].equations, {"continuity": 0, "momentum_x": 0, "momentum_y": 0}, geom["time_rect"], - {**train_dataloader_cfg, "batch_size": NPOINT_PDE * NTIME_PDE}, + {"batch_size": NPOINT_PDE * NTIME_PDE}, ppsci.loss.MSELoss("sum"), evenly=True, weight_dict=cfg.TRAIN.weight.pde, # (1) @@ -72,7 +66,7 @@ def train(cfg: DictConfig): {"u": lambda out: out["u"], "v": lambda out: out["v"]}, {"u": 1, "v": 0}, geom["time_rect"], - {**train_dataloader_cfg, "batch_size": NPOINT_TOP * NTIME_TOP}, + {"batch_size": NPOINT_TOP * NTIME_TOP}, ppsci.loss.MSELoss("sum"), criteria=lambda t, x, y: np.isclose(y, 0.05), name="BC_top", @@ -81,7 +75,7 @@ def train(cfg: DictConfig): {"u": lambda out: out["u"], "v": lambda out: out["v"]}, {"u": 0, "v": 0}, geom["time_rect"], - {**train_dataloader_cfg, "batch_size": NPOINT_DOWN * NTIME_DOWN}, + {"batch_size": NPOINT_DOWN * NTIME_DOWN}, ppsci.loss.MSELoss("sum"), criteria=lambda t, x, y: np.isclose(y, -0.05), name="BC_down", @@ -90,7 +84,7 @@ def train(cfg: DictConfig): {"u": lambda out: out["u"], "v": lambda out: out["v"]}, {"u": 0, "v": 0}, geom["time_rect"], - {**train_dataloader_cfg, "batch_size": NPOINT_LEFT * NTIME_LEFT}, + {"batch_size": NPOINT_LEFT * NTIME_LEFT}, ppsci.loss.MSELoss("sum"), criteria=lambda t, x, y: np.isclose(x, -0.05), name="BC_left", @@ -99,7 +93,7 @@ def train(cfg: DictConfig): {"u": lambda out: out["u"], "v": lambda out: out["v"]}, {"u": 0, "v": 0}, geom["time_rect"], - {**train_dataloader_cfg, "batch_size": NPOINT_RIGHT * NTIME_RIGHT}, + {"batch_size": NPOINT_RIGHT * NTIME_RIGHT}, ppsci.loss.MSELoss("sum"), criteria=lambda t, x, y: np.isclose(x, 0.05), name="BC_right", @@ -108,7 +102,7 @@ def train(cfg: DictConfig): {"u": lambda out: out["u"], "v": lambda out: out["v"]}, {"u": 0, "v": 0}, geom["time_rect"], - {**train_dataloader_cfg, "batch_size": NPOINT_IC * NTIME_IC}, + {"batch_size": NPOINT_IC * NTIME_IC}, ppsci.loss.MSELoss("sum"), evenly=True, name="IC", @@ -139,10 +133,8 @@ def train(cfg: DictConfig): {"momentum_x": 0, "continuity": 0, "momentum_y": 0}, geom["time_rect"], { - "dataset": "NamedArrayDataset", "total_size": NPOINT_EVAL, "batch_size": cfg.EVAL.batch_size.residual_validator, - "sampler": {"name": "BatchSampler"}, }, ppsci.loss.MSELoss("sum"), evenly=True, @@ -247,10 +239,8 @@ def evaluate(cfg: DictConfig): {"momentum_x": 0, "continuity": 0, "momentum_y": 0}, geom["time_rect"], { - "dataset": "NamedArrayDataset", "total_size": NPOINT_EVAL, "batch_size": cfg.EVAL.batch_size.residual_validator, - "sampler": {"name": "BatchSampler"}, }, ppsci.loss.MSELoss("sum"), evenly=True, diff --git a/ppsci/constraint/boundary_constraint.py b/ppsci/constraint/boundary_constraint.py index 2945c71b6..7c9517d83 100644 --- a/ppsci/constraint/boundary_constraint.py +++ b/ppsci/constraint/boundary_constraint.py @@ -98,6 +98,11 @@ def __init__( if isinstance(criteria, str): criteria = eval(criteria) + if "dataset" not in dataloader_cfg: + dataloader_cfg["dataset"] = {"name": "IterableNamedArrayDataset"} + if "iters_per_epoch" not in dataloader_cfg: + dataloader_cfg["iters_per_epoch"] = 1 + # prepare input input = geom.sample_boundary( dataloader_cfg["batch_size"] * dataloader_cfg["iters_per_epoch"], diff --git a/ppsci/constraint/initial_constraint.py b/ppsci/constraint/initial_constraint.py index 7193ca82f..b62aa628e 100644 --- a/ppsci/constraint/initial_constraint.py +++ b/ppsci/constraint/initial_constraint.py @@ -104,6 +104,11 @@ def __init__( if isinstance(criteria, str): criteria = eval(criteria) + if "dataset" not in dataloader_cfg: + dataloader_cfg["dataset"] = {"name": "IterableNamedArrayDataset"} + if "iters_per_epoch" not in dataloader_cfg: + dataloader_cfg["iters_per_epoch"] = 1 + # prepare input input = geom.sample_initial_interior( dataloader_cfg["batch_size"] * dataloader_cfg["iters_per_epoch"], diff --git a/ppsci/constraint/integral_constraint.py b/ppsci/constraint/integral_constraint.py index f9cb22674..4bf7fc29b 100644 --- a/ppsci/constraint/integral_constraint.py +++ b/ppsci/constraint/integral_constraint.py @@ -99,6 +99,11 @@ def __init__( if isinstance(criteria, str): criteria = eval(criteria) + if "dataset" not in dataloader_cfg: + dataloader_cfg["dataset"] = {"name": "IterableNamedArrayDataset"} + if "iters_per_epoch" not in dataloader_cfg: + dataloader_cfg["iters_per_epoch"] = 1 + # prepare input input_list: List[Dict[str, np.ndarray]] = [] for _ in range( diff --git a/ppsci/constraint/interior_constraint.py b/ppsci/constraint/interior_constraint.py index d9e52d060..c9ff52eda 100644 --- a/ppsci/constraint/interior_constraint.py +++ b/ppsci/constraint/interior_constraint.py @@ -101,6 +101,11 @@ def __init__( if isinstance(criteria, str): criteria = eval(criteria) + if "dataset" not in dataloader_cfg: + dataloader_cfg["dataset"] = {"name": "IterableNamedArrayDataset"} + if "iters_per_epoch" not in dataloader_cfg: + dataloader_cfg["iters_per_epoch"] = 1 + # prepare input input = geom.sample_interior( dataloader_cfg["batch_size"] * dataloader_cfg["iters_per_epoch"], diff --git a/ppsci/constraint/periodic_constraint.py b/ppsci/constraint/periodic_constraint.py index 8b50e21aa..b2f321279 100644 --- a/ppsci/constraint/periodic_constraint.py +++ b/ppsci/constraint/periodic_constraint.py @@ -84,6 +84,11 @@ def __init__( if isinstance(criteria, str): criteria = eval(criteria) + if "dataset" not in dataloader_cfg: + dataloader_cfg["dataset"] = {"name": "IterableNamedArrayDataset"} + if "iters_per_epoch" not in dataloader_cfg: + dataloader_cfg["iters_per_epoch"] = 1 + if dataloader_cfg["batch_size"] % 2 > 0: raise ValueError( f"batch_size({dataloader_cfg['sampler']['batch_size']}) " diff --git a/ppsci/constraint/supervised_constraint.py b/ppsci/constraint/supervised_constraint.py index 84b881622..5fa10030a 100644 --- a/ppsci/constraint/supervised_constraint.py +++ b/ppsci/constraint/supervised_constraint.py @@ -60,6 +60,11 @@ def __init__( output_expr: Optional[Dict[str, Callable]] = None, name: str = "Sup", ): + if "dataset" not in dataloader_cfg: + dataloader_cfg["dataset"] = {"name": "IterableNamedArrayDataset"} + if "iters_per_epoch" not in dataloader_cfg: + dataloader_cfg["iters_per_epoch"] = 1 + # build dataset _dataset = dataset.build_dataset(dataloader_cfg["dataset"]) diff --git a/ppsci/data/__init__.py b/ppsci/data/__init__.py index 7a27f891b..9dc0b96ee 100644 --- a/ppsci/data/__init__.py +++ b/ppsci/data/__init__.py @@ -67,9 +67,9 @@ def build_dataloader(_dataset, cfg): cfg = copy.deepcopy(cfg) # build sampler - sampler_cfg = cfg.pop("sampler", None) - if sampler_cfg is not None: - sampler_cls = sampler_cfg.pop("name") + batch_sampler_cfg = cfg.pop("sampler", None) + if batch_sampler_cfg is not None: + sampler_cls = batch_sampler_cfg.pop("name") if sampler_cls == "BatchSampler": if world_size > 1: @@ -79,8 +79,8 @@ def build_dataloader(_dataset, cfg): f"'BatchSampler' when world_size({world_size}) > 1." ) - sampler_cfg["batch_size"] = cfg["batch_size"] - sampler = getattr(io, sampler_cls)(_dataset, **sampler_cfg) + batch_sampler_cfg["batch_size"] = cfg["batch_size"] + batch_sampler = getattr(io, sampler_cls)(_dataset, **batch_sampler_cfg) else: if cfg["batch_size"] != 1: raise ValueError( @@ -89,7 +89,7 @@ def build_dataloader(_dataset, cfg): logger.warning( "`batch_size` is set to 1 as neither sampler config nor batch_size is set." ) - sampler = None + batch_sampler = None # build collate_fn if specified batch_transforms_cfg = cfg.pop("batch_transforms", None) @@ -122,8 +122,8 @@ def build_dataloader(_dataset, cfg): dataloader_ = pgl_data.Dataloader( dataset=_dataset, batch_size=cfg["batch_size"], - drop_last=sampler_cfg.get("drop_last", False), - shuffle=sampler_cfg.get("shuffle", False), + drop_last=batch_sampler_cfg.get("drop_last", False), + shuffle=batch_sampler_cfg.get("shuffle", False), num_workers=cfg.get("num_workers", _DEFAULT_NUM_WORKERS), collate_fn=collate_fn, ) @@ -131,7 +131,7 @@ def build_dataloader(_dataset, cfg): dataloader_ = io.DataLoader( dataset=_dataset, places=device.get_device(), - batch_sampler=sampler, + batch_sampler=batch_sampler, collate_fn=collate_fn, num_workers=cfg.get("num_workers", _DEFAULT_NUM_WORKERS), use_shared_memory=cfg.get("use_shared_memory", False), @@ -140,8 +140,8 @@ def build_dataloader(_dataset, cfg): if len(dataloader_) == 0: raise ValueError( - f"batch_size({sampler_cfg['batch_size']}) should not bigger than number of " - f"samples({len(_dataset)}) when drop_last is {sampler_cfg.get('drop_last', False)}." + f"batch_size({batch_sampler_cfg['batch_size']}) should not bigger than number of " + f"samples({len(_dataset)}) when drop_last is {batch_sampler_cfg.get('drop_last', False)}." ) return dataloader_ diff --git a/ppsci/data/dataset/__init__.py b/ppsci/data/dataset/__init__.py index bbddf17b2..9a1823aac 100644 --- a/ppsci/data/dataset/__init__.py +++ b/ppsci/data/dataset/__init__.py @@ -36,6 +36,9 @@ from ppsci.utils import logger if TYPE_CHECKING: + from typing import Any + from typing import Dict + from paddle import io __all__ = [ @@ -60,11 +63,11 @@ ] -def build_dataset(cfg) -> "io.Dataset": +def build_dataset(cfg: "Dict[str, Any]") -> "io.Dataset": """Build dataset Args: - cfg (List[AttrDict]): dataset config list. + cfg (Dict[str, Any]): Dataset config. Returns: Dict[str, io.Dataset]: dataset. diff --git a/ppsci/validate/geo_validator.py b/ppsci/validate/geo_validator.py index 6741baddc..2300d63cf 100644 --- a/ppsci/validate/geo_validator.py +++ b/ppsci/validate/geo_validator.py @@ -88,6 +88,11 @@ def __init__( self.input_keys = geom.dim_keys self.output_keys = tuple(label_dict.keys()) + if "dataset" not in dataloader_cfg: + dataloader_cfg["dataset"] = "NamedArrayDataset" + if "sampler" not in dataloader_cfg: + dataloader_cfg["sampler"] = {"name": "BatchSampler"} + nx = dataloader_cfg["total_size"] self.num_timestamps = 1 # TODO(sensen): simplify code below diff --git a/ppsci/validate/sup_validator.py b/ppsci/validate/sup_validator.py index a88a02af5..e192efea1 100644 --- a/ppsci/validate/sup_validator.py +++ b/ppsci/validate/sup_validator.py @@ -70,6 +70,11 @@ def __init__( ): self.output_expr = output_expr + if "dataset" not in dataloader_cfg: + dataloader_cfg["dataset"] = {"name": "NamedArrayDataset"} + if "sampler" not in dataloader_cfg: + dataloader_cfg["sampler"] = {"name": "BatchSampler"} + # build dataset _dataset = dataset.build_dataset(dataloader_cfg["dataset"])