You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
import torch as t
from torch import nn
from pipeline import fate_torch_hook
from pipeline.component import HomoNN
from pipeline.backend.pipeline import PipeLine
from pipeline.component import Reader, Evaluation, DataTransform
from pipeline.interface import Data, Model
import torch
fate_torch_hook(t)
import os
fate_project_path = os.path.abspath('/data/prjects/fate')
guest_0 = 10000
host_1 = 10000
pipeline = PipeLine().set_initiator(role='guest', party_id=guest_0).set_roles(guest=guest_0, host=host_1,
arbiter=guest_0)
data_0 = {"name": "imdb", "namespace": "experiment"}
data_path = '/data/projects/fate/examples/data/train_for_fl.csv'
pipeline.bind_table(name=data_0['name'], namespace=data_0['namespace'], path=data_path)
pipeline.bind_table(name=data_0['name'], namespace=data_0['namespace'], path=data_path)
reader_0 = Reader(name="reader_0")
reader_0.get_party_instance(role='guest', party_id=guest_0).component_param(table=data_0)
reader_0.get_party_instance(role='host', party_id=host_1).component_param(table=data_0)
reader_1 = Reader(name="reader_1")
reader_1.get_party_instance(role='guest', party_id=guest_0).component_param(table=data_0)
reader_1.get_party_instance(role='host', party_id=host_1).component_param(table=data_0)
from pipeline.component.homo_nn import DatasetParam, TrainerParam
model = t.nn.Sequential(
t.nn.CustModel(module_name='bert_', class_name='BertClassifier')
)
nn_component = HomoNN(name='nn_0',
model=model,
loss=t.nn.BCELoss(),
optimizer = t.optim.Adam(lr=0.001, weight_decay=0.001),
dataset=DatasetParam(dataset_name='nlp_tokenizer', tokenizer_name_or_path="bert-base-uncased"), # 使用自定义的dataset
trainer=TrainerParam(trainer_name='fedavg_trainer', epochs=2, batch_size=16, data_loader_worker=8, cuda="cuda:0"),
torch_seed=100
)
pipeline.add_component(reader_0)
pipeline.add_component(reader_1)
pipeline.add_component(nn_component, data=Data(train_data=reader_0.output.data, validate_data=reader_1.output.data))
pipeline.add_component(Evaluation(name='eval_0', eval_type='multi'), data=Data(data=nn_component.output.data))
pipeline.compile()
pipeline.fit()
所以重点是把evaluation改成了“multi”,任务在evaluation报错了:
[ERROR] [2024-07-16 16:31:08,499] [202407161623325775800] [38874:139965857994560] - [task_executor.run] [line:266]: 'numpy.float64' object cannot be interpreted as an integer
Traceback (most recent call last):
File "/data/projects/fate/fateflow/python/fate_flow/worker/task_executor.py", line 210, in run
cpn_output = run_object.run(cpn_input)
File "/data/projects/fate/fate/python/federatedml/model_base.py", line 239, in run
self._run(cpn_input=cpn_input)
File "/data/projects/fate/fate/python/federatedml/model_base.py", line 315, in _run
this_data_output = func(*real_param)
File "/data/projects/fate/fate/python/federatedml/evaluation/evaluation.py", line 400, in fit
return self.callback_metric_data(
File "/data/projects/fate/fate/python/federatedml/evaluation/evaluation.py", line 1001, in callback_metric_data
self.__save_pr_curve(precision_recall, data_type)
File "/data/projects/fate/fate/python/federatedml/evaluation/evaluation.py", line 697, in __save_pr_curve
pos_recall_score, recall_cuts = self.__multi_class_label_padding(
File "/data/projects/fate/fate/python/federatedml/evaluation/evaluation.py", line 478, in __multi_class_label_padding
for i in range(label_num):
TypeError: 'numpy.float64' object cannot be interpreted as an integer
我在用自定义算法跑联邦下的bert,参考教程:https://fate.readthedocs.io/en/develop-1.11.1/tutorial/pipeline/nn_tutorial/Bert-example/#dataset-imdb-sentimental
其中有一些细节我已经做了修改,IMDB的示例数据已经跑通,所以我进一步想跑一下文本多分类,使用了一个医学数据集:https://www.kaggle.com/datasets/jpmiller/layoutlm, 已经处理成和IMDB一样的格式:
用notebook提交任务:
所以重点是把evaluation改成了“multi”,任务在evaluation报错了:
[ERROR] [2024-07-16 16:31:08,499] [202407161623325775800] [38874:139965857994560] - [task_executor.run] [line:266]: 'numpy.float64' object cannot be interpreted as an integer
Traceback (most recent call last):
File "/data/projects/fate/fateflow/python/fate_flow/worker/task_executor.py", line 210, in run
cpn_output = run_object.run(cpn_input)
File "/data/projects/fate/fate/python/federatedml/model_base.py", line 239, in run
self._run(cpn_input=cpn_input)
File "/data/projects/fate/fate/python/federatedml/model_base.py", line 315, in _run
this_data_output = func(*real_param)
File "/data/projects/fate/fate/python/federatedml/evaluation/evaluation.py", line 400, in fit
return self.callback_metric_data(
File "/data/projects/fate/fate/python/federatedml/evaluation/evaluation.py", line 1001, in callback_metric_data
self.__save_pr_curve(precision_recall, data_type)
File "/data/projects/fate/fate/python/federatedml/evaluation/evaluation.py", line 697, in __save_pr_curve
pos_recall_score, recall_cuts = self.__multi_class_label_padding(
File "/data/projects/fate/fate/python/federatedml/evaluation/evaluation.py", line 478, in __multi_class_label_padding
for i in range(label_num):
TypeError: 'numpy.float64' object cannot be interpreted as an integer
顺着报错信息我看到/data/projects/fate/fate/python/federatedml/evaluation/evaluation.py这个文件,核心是label_num是float不是int,因此我修改了evaluation.py, 将label_num强制转成int,任务没有再报错,这里是bug吗?
The text was updated successfully, but these errors were encountered: