Internal package documentation#

skyplane.replicate.replicator_client module#

class skyplane.replicate.replicator_client.ReplicatorClient(topology, gateway_docker_image='public.ecr.aws/s6m1p0n8/skyplane:edge', aws_instance_class='m5.4xlarge', azure_instance_class='Standard_D2_v5', gcp_instance_class='n2-standard-16', gcp_use_premium_network=True)[source]#

Bases: object

check_error_logs()[source]#
Return type:

Dict[str, List[str]]

deprovision_gateways()[source]#
get_chunk_status_log_df()[source]#
Return type:

DataFrame

monitor_transfer(job, show_spinner=False, log_interval_s=None, log_to_file=True, time_limit_seconds=None, cleanup_gateway=True, save_log=True, write_profile=False, write_socket_profile=False, copy_gateway_logs=False, multipart=False)[source]#
Return type:

TransferStats

provision_gateways(reuse_instances=False, log_dir=None, authorize_ssh_pub_key=None, use_bbr=True, use_compression=True, use_e2ee=True, use_socket_tls=False, aws_use_spot_instances=False, azure_use_spot_instances=False, gcp_use_spot_instances=False)[source]#
run_replication_plan(job, multipart_enabled, multipart_min_threshold_mb, multipart_chunk_size_mb, multipart_max_chunks)[source]#
Return type:

ReplicationJob

static verify_transfer_prefix(job, dest_prefix)[source]#

Check that all objects to copy are present in the destination

class skyplane.replicate.replicator_client.TransferStats(monitor_status, total_runtime_s=None, throughput_gbits=None, errors=None)[source]#

Bases: object

classmethod empty()[source]#
errors: Optional[Dict[str, List[str]]] = None#
monitor_status: str#
throughput_gbits: Optional[float] = None#
to_dict()[source]#
Return type:

Dict[str, Optional[Any]]

total_runtime_s: Optional[float] = None#
skyplane.replicate.replicator_client.refresh_instance_list(provider, region_list=(), instance_filter=None, n=-1)[source]#
Return type:

Dict[str, List[Server]]

skyplane.chunk module#

class skyplane.chunk.Chunk(src_key, dest_key, chunk_id, chunk_length_bytes, md5_hash=None, file_offset_bytes=None, part_number=None, upload_id=None)[source]#

Bases: object

A Chunk is a contiguous piece of a file (a file may be one or more chunks).

as_dict()[source]#
chunk_id: int#
chunk_length_bytes: int#
dest_key: str#
file_offset_bytes: Optional[int] = None#
static from_dict(d)[source]#
md5_hash: Optional[bytes] = None#
part_number: Optional[int] = None#
src_key: str#
to_wire_header(n_chunks_left_on_socket, wire_length, is_compressed=False)[source]#
upload_id: Optional[str] = None#
class skyplane.chunk.ChunkRequest(chunk, src_region, dst_region, src_type, dst_type, src_random_size_mb=None, src_object_store_bucket=None, dst_object_store_bucket=None)[source]#

Bases: object

A ChunkRequest stores all local state in the Gateway pertaining to a ChunkRequest.

as_dict()[source]#
chunk: Chunk#
dst_object_store_bucket: Optional[str] = None#
dst_region: str#
dst_type: str#
static from_dict(in_dict)[source]#
src_object_store_bucket: Optional[str] = None#
src_random_size_mb: Optional[int] = None#
src_region: str#
src_type: str#
class skyplane.chunk.ChunkState(value)[source]#

Bases: Enum

An enumeration.

download_in_progress = 3#
download_queued = 2#
downloaded = 4#
failed = 8#
static from_str(s)[source]#
registered = 1#
upload_complete = 7#
upload_in_progress = 6#
upload_queued = 5#
class skyplane.chunk.WireProtocolHeader(chunk_id, data_len, is_compressed, n_chunks_left_on_socket)[source]#

Bases: object

Lightweight wire protocol header for chunk transfers along socket.

chunk_id: int#
data_len: int#
static from_bytes(data)[source]#
static from_socket(sock)[source]#
is_compressed: bool#
static length_bytes()[source]#
static magic_hex()[source]#
n_chunks_left_on_socket: int#
static protocol_version()[source]#
to_bytes()[source]#
to_socket(sock)[source]#

skyplane.config module#

class skyplane.config.SkyplaneConfig(aws_enabled, azure_enabled, gcp_enabled, azure_principal_id=None, azure_subscription_id=None, azure_client_id=None, gcp_project_id=None, anon_clientid=None)[source]#

Bases: object

anon_clientid: Optional[str] = None#
aws_enabled: bool#
azure_client_id: Optional[str] = None#
azure_enabled: bool#
azure_principal_id: Optional[str] = None#
azure_subscription_id: Optional[str] = None#
check_config()[source]#
static default_config()[source]#
Return type:

SkyplaneConfig

gcp_enabled: bool#
gcp_project_id: Optional[str] = None#
get_flag(flag_name)[source]#
static load_config(path)[source]#

Load from a config file.

Return type:

SkyplaneConfig

set_flag(flag_name, value)[source]#
to_config_file(path)[source]#
valid_flags()[source]#
skyplane.config._map_type(value, val_type)[source]#

skyplane.exceptions module#

exception skyplane.exceptions.BadConfigException[source]#

Bases: Exception

args#
with_traceback()#

Exception.with_traceback(tb) – set self.__traceback__ to tb and return self.

exception skyplane.exceptions.ChecksumMismatchException[source]#

Bases: SkyplaneException

args#
pretty_print_str()[source]#
with_traceback()#

Exception.with_traceback(tb) – set self.__traceback__ to tb and return self.

exception skyplane.exceptions.InsufficientVCPUException[source]#

Bases: SkyplaneException

args#
pretty_print_str()[source]#
with_traceback()#

Exception.with_traceback(tb) – set self.__traceback__ to tb and return self.

exception skyplane.exceptions.MissingBucketException[source]#

Bases: SkyplaneException

args#
pretty_print_str()[source]#
with_traceback()#

Exception.with_traceback(tb) – set self.__traceback__ to tb and return self.

exception skyplane.exceptions.MissingObjectException[source]#

Bases: SkyplaneException

args#
pretty_print_str()[source]#
with_traceback()#

Exception.with_traceback(tb) – set self.__traceback__ to tb and return self.

exception skyplane.exceptions.NoSuchObjectException[source]#

Bases: Exception

args#
with_traceback()#

Exception.with_traceback(tb) – set self.__traceback__ to tb and return self.

exception skyplane.exceptions.SkyplaneException[source]#

Bases: Exception

args#
pretty_print_str()[source]#
with_traceback()#

Exception.with_traceback(tb) – set self.__traceback__ to tb and return self.

exception skyplane.exceptions.TransferFailedException(message, failed_objects=None)[source]#

Bases: SkyplaneException

args#
pretty_print_str()[source]#
with_traceback()#

Exception.with_traceback(tb) – set self.__traceback__ to tb and return self.