dpdispatcher.machines package

Machines.

Submodules

dpdispatcher.machines.distributed_shell module

class dpdispatcher.machines.distributed_shell.DistributedShell(*args, **kwargs)

Bases: Machine

Methods

do_submit(job)	Submit th job to yarn using distributed shell.
get_exit_code(job)	Get exit code of the job.
kill(job)	Kill the job.
resources_arginfo()	Generate the resources arginfo.
resources_subfields()	Generate the resources subfields.

arginfo
bind_context
check_finish_tag
check_if_recover
check_status
default_resources
deserialize
gen_command_env_cuda_devices
gen_script
gen_script_command
gen_script_custom_flags_lines
gen_script_end
gen_script_env
gen_script_header
gen_script_run_command
gen_script_wait
load_from_dict
load_from_json
load_from_yaml
serialize
sub_script_cmd
sub_script_head

check_finish_tag(job)

check_status(job)

do_submit(job)

Submit th job to yarn using distributed shell.

Parameters:

jobJob class instance

job to be submitted

Returns:

job_id: string

submit process id

gen_script_end(job)

gen_script_env(job)

gen_script_header(job)

dpdispatcher.machines.dp_cloud_server module

class dpdispatcher.machines.dp_cloud_server.Bohrium(*args, **kwargs)

Bases: Machine

Methods

do_submit(job)	Submit a single job, assuming that no job is running there.
get_exit_code(job)	Get exit code of the job.
kill(job)	Kill the job.
resources_arginfo()	Generate the resources arginfo.
resources_subfields()	Generate the resources subfields.

arginfo
bind_context
check_finish_tag
check_if_recover
check_status
default_resources
deserialize
gen_command_env_cuda_devices
gen_local_script
gen_script
gen_script_command
gen_script_custom_flags_lines
gen_script_end
gen_script_env
gen_script_header
gen_script_run_command
gen_script_wait
load_from_dict
load_from_json
load_from_yaml
map_dp_job_state
serialize
sub_script_cmd
sub_script_head

alias: Tuple[str, ...] = ('Lebesgue', 'DpCloudServer')

check_finish_tag(job)

check_if_recover(submission)

check_status(job)

do_submit(job)

Submit a single job, assuming that no job is running there.

gen_local_script(job)

gen_script(job)

gen_script_header(job)

get_exit_code(job) → int

Get exit code of the job.

Parameters:

jobJob

job

kill(job)

Kill the job.

Parameters:

jobJob

job

static map_dp_job_state(status, exit_code, ignore_exit_code=True)

dpdispatcher.machines.dp_cloud_server.DpCloudServer

alias of Bohrium

dpdispatcher.machines.dp_cloud_server.Lebesgue

alias of Bohrium

dpdispatcher.machines.fugaku module

class dpdispatcher.machines.fugaku.Fugaku(*args, **kwargs)

Bases: Machine

Methods

do_submit(job)	Submit a single job, assuming that no job is running there.
get_exit_code(job)	Get exit code of the job.
kill(job)	Kill the job.
resources_arginfo()	Generate the resources arginfo.
resources_subfields()	Generate the resources subfields.

arginfo
bind_context
check_finish_tag
check_if_recover
check_status
default_resources
deserialize
gen_command_env_cuda_devices
gen_script
gen_script_command
gen_script_custom_flags_lines
gen_script_end
gen_script_env
gen_script_header
gen_script_run_command
gen_script_wait
load_from_dict
load_from_json
load_from_yaml
serialize
sub_script_cmd
sub_script_head

check_finish_tag(job)

check_status(job)

default_resources(resources)

do_submit(job)

Submit a single job, assuming that no job is running there.

gen_script(job)

gen_script_header(job)

dpdispatcher.machines.lsf module

class dpdispatcher.machines.lsf.LSF(*args, **kwargs)

Bases: Machine

LSF batch.

Methods

default_resources(resources)
get_exit_code(job)	Get exit code of the job.
kill(job)	Kill the job.
resources_arginfo()	Generate the resources arginfo.
resources_subfields()	Generate the resources subfields.

arginfo
bind_context
check_finish_tag
check_if_recover
check_status
deserialize
do_submit
gen_command_env_cuda_devices
gen_script
gen_script_command
gen_script_custom_flags_lines
gen_script_end
gen_script_env
gen_script_header
gen_script_run_command
gen_script_wait
load_from_dict
load_from_json
load_from_yaml
serialize
sub_script_cmd
sub_script_head

check_finish_tag(job)

check_status(**kwargs)

default_resources(resources)

do_submit(**kwargs)

Submit a single job, assuming that no job is running there.

gen_script(job)

gen_script_header(job)

kill(job)

Kill the job.

Parameters:

jobJob

job

classmethod resources_subfields() → List[Argument]

Generate the resources subfields.

Returns:

list[Argument]

resources subfields

sub_script_cmd(res)

sub_script_head(res)

dpdispatcher.machines.openapi module

class dpdispatcher.machines.openapi.OpenAPI(*args, **kwargs)

Bases: Machine

Methods

do_submit(job)	Submit a single job, assuming that no job is running there.
get_exit_code(job)	Get exit code of the job.
kill(job)	Kill the job.
resources_arginfo()	Generate the resources arginfo.
resources_subfields()	Generate the resources subfields.

arginfo
bind_context
check_finish_tag
check_if_recover
check_status
default_resources
deserialize
gen_command_env_cuda_devices
gen_local_script
gen_script
gen_script_command
gen_script_custom_flags_lines
gen_script_end
gen_script_env
gen_script_header
gen_script_run_command
gen_script_wait
load_from_dict
load_from_json
load_from_yaml
map_dp_job_state
serialize
sub_script_cmd
sub_script_head

check_finish_tag(job)

check_if_recover(submission)

check_status(job)

do_submit(job)

Submit a single job, assuming that no job is running there.

gen_local_script(job)

gen_script(job)

gen_script_header(job)

get_exit_code(job)

Get exit code of the job.

Parameters:

jobJob

job

Returns:

int

exit code

kill(job)

Kill the job.

Parameters:

jobJob

job

static map_dp_job_state(status, exit_code, ignore_exit_code=True)

dpdispatcher.machines.pbs module

class dpdispatcher.machines.pbs.PBS(*args, **kwargs)

Bases: Machine

Methods

do_submit(job)	Submit a single job, assuming that no job is running there.
get_exit_code(job)	Get exit code of the job.
kill(job)	Kill the job.
resources_arginfo()	Generate the resources arginfo.
resources_subfields()	Generate the resources subfields.

arginfo
bind_context
check_finish_tag
check_if_recover
check_status
default_resources
deserialize
gen_command_env_cuda_devices
gen_script
gen_script_command
gen_script_custom_flags_lines
gen_script_end
gen_script_env
gen_script_header
gen_script_run_command
gen_script_wait
load_from_dict
load_from_json
load_from_yaml
serialize
sub_script_cmd
sub_script_head

check_finish_tag(job)

check_status(job)

default_resources(resources)

do_submit(job)

Submit a single job, assuming that no job is running there.

gen_script(job)

gen_script_header(job)

kill(job)

Kill the job.

Parameters:

jobJob

job

class dpdispatcher.machines.pbs.SGE(*args, **kwargs)

Bases: PBS

Methods

do_submit(job)	Submit a single job, assuming that no job is running there.
get_exit_code(job)	Get exit code of the job.
kill(job)	Kill the job.
resources_arginfo()	Generate the resources arginfo.
resources_subfields()	Generate the resources subfields.

arginfo
bind_context
check_finish_tag
check_if_recover
check_status
default_resources
deserialize
gen_command_env_cuda_devices
gen_script
gen_script_command
gen_script_custom_flags_lines
gen_script_end
gen_script_env
gen_script_header
gen_script_run_command
gen_script_wait
load_from_dict
load_from_json
load_from_yaml
serialize
sub_script_cmd
sub_script_head

check_finish_tag(job)

check_status(job)

default_resources(resources)

do_submit(job)

Submit a single job, assuming that no job is running there.

gen_script_header(job)

class dpdispatcher.machines.pbs.Torque(*args, **kwargs)

Bases: PBS

Methods

do_submit(job)	Submit a single job, assuming that no job is running there.
get_exit_code(job)	Get exit code of the job.
kill(job)	Kill the job.
resources_arginfo()	Generate the resources arginfo.
resources_subfields()	Generate the resources subfields.

arginfo
bind_context
check_finish_tag
check_if_recover
check_status
default_resources
deserialize
gen_command_env_cuda_devices
gen_script
gen_script_command
gen_script_custom_flags_lines
gen_script_end
gen_script_env
gen_script_header
gen_script_run_command
gen_script_wait
load_from_dict
load_from_json
load_from_yaml
serialize
sub_script_cmd
sub_script_head

check_status(job)

gen_script_header(job)

dpdispatcher.machines.shell module

class dpdispatcher.machines.shell.Shell(*args, **kwargs)

Bases: Machine

Methods

do_submit(job)	Submit a single job, assuming that no job is running there.
get_exit_code(job)	Get exit code of the job.
kill(job)	Kill the job.
resources_arginfo()	Generate the resources arginfo.
resources_subfields()	Generate the resources subfields.

arginfo
bind_context
check_finish_tag
check_if_recover
check_status
default_resources
deserialize
gen_command_env_cuda_devices
gen_script
gen_script_command
gen_script_custom_flags_lines
gen_script_end
gen_script_env
gen_script_header
gen_script_run_command
gen_script_wait
load_from_dict
load_from_json
load_from_yaml
serialize
sub_script_cmd
sub_script_head

check_finish_tag(job)

check_status(job)

default_resources(resources)

do_submit(job)

Submit a single job, assuming that no job is running there.

gen_script(job)

gen_script_header(job)

kill(job)

Kill the job.

Parameters:

jobJob

job

dpdispatcher.machines.slurm module

class dpdispatcher.machines.slurm.Slurm(*args, **kwargs)

Bases: Machine

Methods

get_exit_code(job)	Get exit code of the job.
kill(job)	Kill the job.
resources_arginfo()	Generate the resources arginfo.
resources_subfields()	Generate the resources subfields.

arginfo
bind_context
check_finish_tag
check_if_recover
check_status
default_resources
deserialize
do_submit
gen_command_env_cuda_devices
gen_script
gen_script_command
gen_script_custom_flags_lines
gen_script_end
gen_script_env
gen_script_header
gen_script_run_command
gen_script_wait
load_from_dict
load_from_json
load_from_yaml
serialize
sub_script_cmd
sub_script_head

check_finish_tag(job)

check_status(**kwargs)

default_resources(resources)

do_submit(**kwargs)

Submit a single job, assuming that no job is running there.

gen_script(job)

gen_script_header(job)

kill(job)

Kill the job.

Parameters:

jobJob

job

classmethod resources_subfields() → List[Argument]

Generate the resources subfields.

Returns:

list[Argument]

resources subfields

class dpdispatcher.machines.slurm.SlurmJobArray(*args, **kwargs)

Bases: Slurm

Slurm with job array enabled for multiple tasks in a job.

Methods

get_exit_code(job)	Get exit code of the job.
kill(job)	Kill the job.
resources_arginfo()	Generate the resources arginfo.
resources_subfields()	Generate the resources subfields.

arginfo
bind_context
check_finish_tag
check_if_recover
check_status
default_resources
deserialize
do_submit
gen_command_env_cuda_devices
gen_script
gen_script_command
gen_script_custom_flags_lines
gen_script_end
gen_script_env
gen_script_header
gen_script_run_command
gen_script_wait
load_from_dict
load_from_json
load_from_yaml
serialize
sub_script_cmd
sub_script_head

check_finish_tag(job)

check_status(**kwargs)

gen_script_command(job)

gen_script_end(job)

gen_script_header(job)

classmethod resources_subfields() → List[Argument]

Generate the resources subfields.

Returns:

list[Argument]

resources subfields