update
This commit is contained in:
liuzx
parent
830db0e6e9
commit
7f95e4ed78
100
inference.py
100
inference.py
|
|
@ -1,85 +1,36 @@
|
|||
|
||||
"""
|
||||
######################## single-dataset inference lenet example ########################
|
||||
This example is a single-dataset inference tutorial.
|
||||
|
||||
######################## Instructions for using the inference environment ########################
|
||||
1、Inference task requires predefined functions
|
||||
(1)Copy single dataset from obs to inference image.
|
||||
function ObsToEnv(obs_data_url, data_dir)
|
||||
|
||||
(2)Copy ckpt file from obs to inference image.
|
||||
function ObsUrlToEnv(obs_ckpt_url, ckpt_url)
|
||||
|
||||
(3)Copy the output result to obs.
|
||||
function EnvToObs(train_dir, obs_train_url)
|
||||
|
||||
3、4 parameters need to be defined.
|
||||
--data_url is the dataset you selected on the Qizhi platform
|
||||
--ckpt_url is the weight file you choose on the Qizhi platform
|
||||
|
||||
--data_url,--ckpt_url,--result_url,--device_target,These 4 parameters must be defined first in a single dataset,
|
||||
otherwise an error will be reported.
|
||||
There is no need to add these parameters to the running parameters of the Qizhi platform,
|
||||
because they are predefined in the background, you only need to define them in your code.
|
||||
|
||||
4、How the dataset is used
|
||||
Inference task uses data_url as the input, and data_dir (ie: '/cache/data') as the calling method
|
||||
of the dataset in the image.
|
||||
For details, please refer to the following sample code.
|
||||
使用注意事项：
|
||||
1、本示例需要用户定义的参数有--multi_data_url,--pretrain_url,--result_url，这3个参数任务中必须定义
|
||||
具体的含义如下：
|
||||
--multi_data_url是启智平台上选择的数据集的obs路径
|
||||
--pretrain_url是启智平台上选择的预训练模型文件的obs路径
|
||||
--result_url是训练结果回传到启智平台的obs路径
|
||||
2、用户需要调用OpenI.py下的DatasetToEnv,PretrainToEnv,UploadToOpenI等函数，来实现数据集、预训练模型文件、训练结果的拷贝和回传
|
||||
"""
|
||||
|
||||
import os
|
||||
import argparse
|
||||
import moxing as mox
|
||||
import mindspore.nn as nn
|
||||
import numpy as np
|
||||
from mindspore import context
|
||||
from mindspore.train.serialization import load_checkpoint, load_param_into_net
|
||||
from mindspore.train import Model
|
||||
from mindspore.nn.metrics import Accuracy
|
||||
from mindspore import Tensor
|
||||
import numpy as np
|
||||
from glob import glob
|
||||
from dataset import create_dataset
|
||||
from config import mnist_cfg as cfg
|
||||
from lenet import LeNet5
|
||||
from OpenI import OpenIMultiDatasetToEnv as DatasetToEnv
|
||||
from OpenI import OpenIPretrainToEnv as PretrainToEnv
|
||||
from OpenI import EnvToOpenI
|
||||
|
||||
### Copy single dataset from obs to inference image ###
|
||||
def ObsToEnv(obs_data_url, data_dir):
|
||||
try:
|
||||
mox.file.copy_parallel(obs_data_url, data_dir)
|
||||
print("Successfully Download {} to {}".format(obs_data_url, data_dir))
|
||||
except Exception as e:
|
||||
print('moxing download {} to {} failed: '.format(obs_data_url, data_dir) + str(e))
|
||||
return
|
||||
### Copy ckpt file from obs to inference image###
|
||||
### To operate on folders, use mox.file.copy_parallel. If copying a file.
|
||||
### Please use mox.file.copy to operate the file, this operation is to operate the file
|
||||
def ObsUrlToEnv(obs_ckpt_url, ckpt_url):
|
||||
try:
|
||||
mox.file.copy(obs_ckpt_url, ckpt_url)
|
||||
print("Successfully Download {} to {}".format(obs_ckpt_url,ckpt_url))
|
||||
except Exception as e:
|
||||
print('moxing download {} to {} failed: '.format(obs_ckpt_url, ckpt_url) + str(e))
|
||||
return
|
||||
### Copy the output result to obs###
|
||||
def EnvToObs(train_dir, obs_train_url):
|
||||
try:
|
||||
mox.file.copy_parallel(train_dir, obs_train_url)
|
||||
print("Successfully Upload {} to {}".format(train_dir,obs_train_url))
|
||||
except Exception as e:
|
||||
print('moxing upload {} to {} failed: '.format(train_dir,obs_train_url) + str(e))
|
||||
return
|
||||
|
||||
### --data_url,--ckpt_url,--result_url,--device_target,These 4 parameters must be defined first in a inference task,
|
||||
### otherwise an error will be reported.
|
||||
### There is no need to add these parameters to the running parameters of the Qizhi platform,
|
||||
### because they are predefined in the background, you only need to define them in your code.
|
||||
parser = argparse.ArgumentParser(description='MindSpore Lenet Example')
|
||||
parser.add_argument('--data_url',
|
||||
parser.add_argument('--multi_data_url',
|
||||
type=str,
|
||||
default= '/cache/data/',
|
||||
help='path where the dataset is saved')
|
||||
parser.add_argument('--ckpt_url',
|
||||
parser.add_argument('--pretrain_url',
|
||||
help='model to save/load',
|
||||
default= '/cache/checkpoint.ckpt')
|
||||
parser.add_argument('--result_url',
|
||||
|
|
@ -93,18 +44,20 @@ if __name__ == "__main__":
|
|||
|
||||
###Initialize the data and result directories in the inference image###
|
||||
data_dir = '/cache/data'
|
||||
pretrain_dir = '/cache/pretrain'
|
||||
result_dir = '/cache/result'
|
||||
ckpt_url = '/cache/checkpoint.ckpt'
|
||||
if not os.path.exists(data_dir):
|
||||
os.makedirs(data_dir)
|
||||
if not os.path.exists(pretrain_dir):
|
||||
os.makedirs(pretrain_dir)
|
||||
if not os.path.exists(result_dir):
|
||||
os.makedirs(result_dir)
|
||||
os.makedirs(result_dir)
|
||||
|
||||
###Copy dataset from obs to inference image
|
||||
ObsToEnv(args.data_url, data_dir)
|
||||
###拷贝数据集到训练环境
|
||||
DatasetToEnv(args.multi_data_url, data_dir)
|
||||
|
||||
###Copy ckpt file from obs to inference image
|
||||
ObsUrlToEnv(args.ckpt_url, ckpt_url)
|
||||
###拷贝预训练模型文件到训练环境
|
||||
PretrainToEnv(args.pretrain_url, pretrain_dir)
|
||||
|
||||
context.set_context(mode=context.GRAPH_MODE, device_target=args.device_target)
|
||||
network = LeNet5(cfg.num_classes)
|
||||
|
|
@ -115,9 +68,9 @@ if __name__ == "__main__":
|
|||
|
||||
print("============== Starting Testing ==============")
|
||||
|
||||
param_dict = load_checkpoint(os.path.join(ckpt_url))
|
||||
param_dict = load_checkpoint(os.path.join(pretrain_dir, "checkpoint_lenet-1_1875.ckpt"))
|
||||
load_param_into_net(network, param_dict)
|
||||
ds_test = create_dataset(os.path.join(data_dir, "test"), batch_size=1).create_dict_iterator()
|
||||
ds_test = create_dataset(os.path.join(data_dir + "/MNISTData", "test"), batch_size=1).create_dict_iterator()
|
||||
data = next(ds_test)
|
||||
images = data["image"].asnumpy()
|
||||
labels = data["label"].asnumpy()
|
||||
|
|
@ -134,6 +87,5 @@ if __name__ == "__main__":
|
|||
with open(file_path, 'a+') as file:
|
||||
file.write(" {}: {:.2f} \n".format("Predicted", predicted[0]))
|
||||
|
||||
###Copy result data from the local running environment back to obs,
|
||||
###and download it in the inference task corresponding to the Qizhi platform
|
||||
EnvToObs(result_dir, args.result_url)
|
||||
###上传训练结果到启智平台
|
||||
EnvToOpenI(result_dir, args.result_url)
|
||||
|
|
@ -1,158 +0,0 @@
|
|||
"""
|
||||
######################## multi-dataset inference lenet example ########################
|
||||
This example is a single-dataset inference tutorial.
|
||||
|
||||
######################## Instructions for using the inference environment ########################
|
||||
1、Inference task requires predefined functions
|
||||
(1)Copy multi dataset from obs to inference image.
|
||||
function MultiObsToEnv(obs_data_url, data_dir)
|
||||
|
||||
(2)Copy ckpt file from obs to inference image.
|
||||
function ObsUrlToEnv(obs_ckpt_url, ckpt_url)
|
||||
|
||||
(3)Copy the output result to obs.
|
||||
function EnvToObs(train_dir, obs_train_url)
|
||||
|
||||
3、5 parameters need to be defined.
|
||||
--data_url is the first dataset you selected on the Qizhi platform
|
||||
--multi_data_url is the multi dataset you selected on the Qizhi platform
|
||||
--ckpt_url is the weight file you choose on the Qizhi platform
|
||||
--result_url is the output
|
||||
|
||||
--data_url,--multi_data_url,--ckpt_url,--result_url,--device_target,These 5 parameters must be defined first in a single dataset,
|
||||
otherwise an error will be reported.
|
||||
There is no need to add these parameters to the running parameters of the Qizhi platform,
|
||||
because they are predefined in the background, you only need to define them in your code.
|
||||
|
||||
4、How the dataset is used
|
||||
Multi-datasets use multi_data_url as input, data_dir + dataset name + file or folder name in the dataset as the
|
||||
calling path of the dataset in the inference image.
|
||||
For example, the calling path of the test folder in the MNIST_Data dataset in this example is
|
||||
data_dir + "/MNIST_Data" +"/test"
|
||||
|
||||
For details, please refer to the following sample code.
|
||||
"""
|
||||
|
||||
import os
|
||||
import argparse
|
||||
import moxing as mox
|
||||
import mindspore.nn as nn
|
||||
from mindspore import context
|
||||
from mindspore.train.serialization import load_checkpoint, load_param_into_net
|
||||
from mindspore.train import Model
|
||||
from mindspore.nn.metrics import Accuracy
|
||||
from mindspore import Tensor
|
||||
import numpy as np
|
||||
from glob import glob
|
||||
from dataset import create_dataset
|
||||
from config import mnist_cfg as cfg
|
||||
from lenet import LeNet5
|
||||
import json
|
||||
|
||||
### Copy multiple datasets from obs to inference image ###
|
||||
def MultiObsToEnv(multi_data_url, data_dir):
|
||||
#--multi_data_url is json data, need to do json parsing for multi_data_url
|
||||
multi_data_json = json.loads(multi_data_url)
|
||||
for i in range(len(multi_data_json)):
|
||||
path = data_dir + "/" + multi_data_json[i]["dataset_name"]
|
||||
if not os.path.exists(path):
|
||||
os.makedirs(path)
|
||||
try:
|
||||
mox.file.copy_parallel(multi_data_json[i]["dataset_url"], path)
|
||||
print("Successfully Download {} to {}".format(multi_data_json[i]["dataset_url"],path))
|
||||
except Exception as e:
|
||||
print('moxing download {} to {} failed: '.format(
|
||||
multi_data_json[i]["dataset_url"], path) + str(e))
|
||||
return
|
||||
### Copy ckpt file from obs to inference image###
|
||||
### To operate on folders, use mox.file.copy_parallel. If copying a file.
|
||||
### Please use mox.file.copy to operate the file, this operation is to operate the file
|
||||
def ObsUrlToEnv(obs_ckpt_url, ckpt_url):
|
||||
try:
|
||||
mox.file.copy(obs_ckpt_url, ckpt_url)
|
||||
print("Successfully Download {} to {}".format(obs_ckpt_url,ckpt_url))
|
||||
except Exception as e:
|
||||
print('moxing download {} to {} failed: '.format(obs_ckpt_url, ckpt_url) + str(e))
|
||||
return
|
||||
### Copy the output result to obs###
|
||||
def EnvToObs(train_dir, obs_train_url):
|
||||
try:
|
||||
mox.file.copy_parallel(train_dir, obs_train_url)
|
||||
print("Successfully Upload {} to {}".format(train_dir,obs_train_url))
|
||||
except Exception as e:
|
||||
print('moxing upload {} to {} failed: '.format(train_dir,obs_train_url) + str(e))
|
||||
return
|
||||
|
||||
|
||||
|
||||
### --data_url,--multi_data_url,--ckpt_url,--result_url,--device_target,These 5 parameters must be defined first in a multi dataset inference task,
|
||||
### otherwise an error will be reported.
|
||||
### There is no need to add these parameters to the running parameters of the Qizhi platform,
|
||||
### because they are predefined in the background, you only need to define them in your code.
|
||||
parser = argparse.ArgumentParser(description='MindSpore Lenet Example')
|
||||
parser.add_argument('--data_url',
|
||||
type=str,
|
||||
default= '/cache/data1/',
|
||||
help='path where the dataset is saved')
|
||||
parser.add_argument('--multi_data_url',
|
||||
type=str,
|
||||
default= '/cache/data/',
|
||||
help='path where the dataset is saved')
|
||||
parser.add_argument('--ckpt_url',
|
||||
help='model to save/load',
|
||||
default= '/cache/checkpoint.ckpt')
|
||||
parser.add_argument('--result_url',
|
||||
help='result folder to save/load',
|
||||
default= '/cache/result/')
|
||||
parser.add_argument('--device_target', type=str, default="Ascend", choices=['Ascend', 'GPU', 'CPU'],
|
||||
help='device where the code will be implemented (default: Ascend)')
|
||||
|
||||
if __name__ == "__main__":
|
||||
args, unknown = parser.parse_known_args()
|
||||
|
||||
###Initialize the data and result directories in the inference image###
|
||||
data_dir = '/cache/data'
|
||||
result_dir = '/cache/result'
|
||||
ckpt_url = '/cache/checkpoint.ckpt'
|
||||
if not os.path.exists(data_dir):
|
||||
os.makedirs(data_dir)
|
||||
if not os.path.exists(result_dir):
|
||||
os.makedirs(result_dir)
|
||||
|
||||
###Copy multiple dataset from obs to inference image
|
||||
MultiObsToEnv(args.multi_data_url, data_dir)
|
||||
|
||||
###Copy ckpt file from obs to inference image
|
||||
ObsUrlToEnv(args.ckpt_url, ckpt_url)
|
||||
|
||||
context.set_context(mode=context.GRAPH_MODE, device_target=args.device_target)
|
||||
network = LeNet5(cfg.num_classes)
|
||||
net_loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean")
|
||||
repeat_size = cfg.epoch_size
|
||||
net_opt = nn.Momentum(network.trainable_params(), cfg.lr, cfg.momentum)
|
||||
model = Model(network, net_loss, net_opt, metrics={"Accuracy"})
|
||||
|
||||
print("============== Starting Testing ==============")
|
||||
|
||||
param_dict = load_checkpoint(os.path.join(ckpt_url))
|
||||
load_param_into_net(network, param_dict)
|
||||
ds_test = create_dataset(os.path.join(data_dir + "/MNISTData", "test"), batch_size=1).create_dict_iterator()
|
||||
data = next(ds_test)
|
||||
images = data["image"].asnumpy()
|
||||
labels = data["label"].asnumpy()
|
||||
print('Tensor:', Tensor(data['image']))
|
||||
output = model.predict(Tensor(data['image']))
|
||||
predicted = np.argmax(output.asnumpy(), axis=1)
|
||||
pred = np.argmax(output.asnumpy(), axis=1)
|
||||
print('predicted:', predicted)
|
||||
print('pred:', pred)
|
||||
|
||||
print(f'Predicted: "{predicted[0]}", Actual: "{labels[0]}"')
|
||||
filename = 'result.txt'
|
||||
file_path = os.path.join(result_dir, filename)
|
||||
with open(file_path, 'a+') as file:
|
||||
file.write(" {}: {:.2f} \n".format("Predicted", predicted[0]))
|
||||
|
||||
###Copy result data from the local running environment back to obs,
|
||||
###and download it in the inference task corresponding to the Qizhi platform
|
||||
EnvToObs(result_dir, args.result_url)
|
||||
|
|
@ -107,7 +107,7 @@ if __name__ == "__main__":
|
|||
time_cb = TimeMonitor(data_size=ds_train.get_dataset_size())
|
||||
|
||||
###假如选择了模型文件，使用pretrain_dir,注意ckpt_url的方式依然保留，你依然可以使用ckpt_url的方式，但是这种方式将会逐渐废弃
|
||||
load_param_into_net(network, load_checkpoint(os.path.join(pretrain_dir, "checkpoint_lenet-2_1875.ckpt")))
|
||||
load_param_into_net(network, load_checkpoint(os.path.join(pretrain_dir, "checkpoint_lenet-1_1875.ckpt")))
|
||||
|
||||
if args.device_target != "Ascend":
|
||||
model = Model(network,
|
||||
|
|
|
|||
14
upload.py
14
upload.py
|
|
@ -1,14 +0,0 @@
|
|||
from mindspore.train.callback import Callback
|
||||
import moxing as mox
|
||||
|
||||
class UploadOutput(Callback):
|
||||
def __init__(self, train_dir, obs_train_url):
|
||||
self.train_dir = train_dir
|
||||
self.obs_train_url = obs_train_url
|
||||
def epoch_end(self,run_context):
|
||||
try:
|
||||
mox.file.copy_parallel(self.train_dir , self.obs_train_url )
|
||||
print("Successfully Upload {} to {}".format(self.train_dir ,self.obs_train_url ))
|
||||
except Exception as e:
|
||||
print('moxing upload {} to {} failed: '.format(self.train_dir ,self.obs_train_url ) + str(e))
|
||||
return
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
from mindspore.train.callback import Callback
|
||||
import os
|
||||
|
||||
class UploadOutput(Callback):
|
||||
def epoch_end(self,run_context):
|
||||
os.system("cd /cache/script_for_grampus/ &&./uploader_for_npu " + "/cache/output/")
|
||||
Loading…
Reference in New Issue