kpt-grpc-demo/google/genomics/v1alpha2/pipelines.proto

// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

syntax = "proto3";

package google.genomics.v1alpha2;

import "google/api/annotations.proto";
import "google/longrunning/operations.proto";
import "google/protobuf/duration.proto";
import "google/protobuf/empty.proto";
import "google/protobuf/timestamp.proto";
import "google/rpc/code.proto";

option cc_enable_arenas = true;
option go_package = "google.golang.org/genproto/googleapis/genomics/v1alpha2;genomics";
option java_multiple_files = true;
option java_outer_classname = "PipelinesProto";
option java_package = "com.google.genomics.v1a";

// A service for running genomics pipelines.
service PipelinesV1Alpha2 {
  // Creates a pipeline that can be run later. Create takes a Pipeline that
  // has all fields other than `pipelineId` populated, and then returns
  // the same pipeline with `pipelineId` populated. This id can be used
  // to run the pipeline.
  //
  // Caller must have WRITE permission to the project.
  rpc CreatePipeline(CreatePipelineRequest) returns (Pipeline) {
    option (google.api.http) = {
      post: "/v1alpha2/pipelines"
      body: "pipeline"
    };
  }

  // Runs a pipeline. If `pipelineId` is specified in the request, then
  // run a saved pipeline. If `ephemeralPipeline` is specified, then run
  // that pipeline once without saving a copy.
  //
  // The caller must have READ permission to the project where the pipeline
  // is stored and WRITE permission to the project where the pipeline will be
  // run, as VMs will be created and storage will be used.
  rpc RunPipeline(RunPipelineRequest) returns (google.longrunning.Operation) {
    option (google.api.http) = {
      post: "/v1alpha2/pipelines:run"
      body: "*"
    };
  }

  // Retrieves a pipeline based on ID.
  //
  // Caller must have READ permission to the project.
  rpc GetPipeline(GetPipelineRequest) returns (Pipeline) {
    option (google.api.http) = {
      get: "/v1alpha2/pipelines/{pipeline_id}"
    };
  }

  // Lists pipelines.
  //
  // Caller must have READ permission to the project.
  rpc ListPipelines(ListPipelinesRequest) returns (ListPipelinesResponse) {
    option (google.api.http) = {
      get: "/v1alpha2/pipelines"
    };
  }

  // Deletes a pipeline based on ID.
  //
  // Caller must have WRITE permission to the project.
  rpc DeletePipeline(DeletePipelineRequest) returns (google.protobuf.Empty) {
    option (google.api.http) = {
      delete: "/v1alpha2/pipelines/{pipeline_id}"
    };
  }

  // Gets controller configuration information. Should only be called
  // by VMs created by the Pipelines Service and not by end users.
  rpc GetControllerConfig(GetControllerConfigRequest)
      returns (ControllerConfig) {
    option (google.api.http) = {
      get: "/v1alpha2/pipelines:getControllerConfig"
    };
  }

  // Sets status of a given operation. Any new timestamps (as determined by
  // description) are appended to TimestampEvents. Should only be called by VMs
  // created by the Pipelines Service and not by end users.
  rpc SetOperationStatus(SetOperationStatusRequest)
      returns (google.protobuf.Empty) {
    option (google.api.http) = {
      put: "/v1alpha2/pipelines:setOperationStatus"
      body: "*"
    };
  }
}

// Describes a Compute Engine resource that is being managed by a running
// [pipeline][google.genomics.v1alpha2.Pipeline].
message ComputeEngine {
  // The instance on which the operation is running.
  string instance_name = 1;

  // The availability zone in which the instance resides.
  string zone = 2;

  // The machine type of the instance.
  string machine_type = 3;

  // The names of the disks that were created for this pipeline.
  repeated string disk_names = 4;
}

// Runtime metadata that will be populated in the
// [runtimeMetadata][google.genomics.v1.OperationMetadata.runtime_metadata]
// field of the Operation associated with a RunPipeline execution.
message RuntimeMetadata {
  // Execution information specific to Google Compute Engine.
  ComputeEngine compute_engine = 1;
}

// The pipeline object. Represents a transformation from a set of input
// parameters to a set of output parameters. The transformation is defined
// as a docker image and command to run within that image. Each pipeline
// is run on a Google Compute Engine VM. A pipeline can be created with the
// `create` method and then later run with the `run` method, or a pipeline can
// be defined and run all at once with the `run` method.
message Pipeline {
  // Required. The project in which to create the pipeline. The caller must have
  // WRITE access.
  string project_id = 1;

  // Required. A user specified pipeline name that does not have to be unique.
  // This name can be used for filtering Pipelines in ListPipelines.
  string name = 2;

  // User-specified description.
  string description = 3;

  // Input parameters of the pipeline.
  repeated PipelineParameter input_parameters = 8;

  // Output parameters of the pipeline.
  repeated PipelineParameter output_parameters = 9;

  // Required. The executor indicates in which environment the pipeline runs.
  oneof executor {
    // Specifies the docker run information.
    DockerExecutor docker = 5;
  }

  // Required. Specifies resource requirements for the pipeline run.
  // Required fields:
  //
  // *
  // [minimumCpuCores][google.genomics.v1alpha2.PipelineResources.minimum_cpu_cores]
  //
  // *
  // [minimumRamGb][google.genomics.v1alpha2.PipelineResources.minimum_ram_gb]
  PipelineResources resources = 6;

  // Unique pipeline id that is generated by the service when CreatePipeline
  // is called. Cannot be specified in the Pipeline used in the
  // CreatePipelineRequest, and will be populated in the response to
  // CreatePipeline and all subsequent Get and List calls. Indicates that the
  // service has registered this pipeline.
  string pipeline_id = 7;
}

// The request to create a pipeline. The pipeline field here should not have
// `pipelineId` populated, as that will be populated by the server.
message CreatePipelineRequest {
  // The pipeline to create. Should not have `pipelineId` populated.
  Pipeline pipeline = 1;
}

// The pipeline run arguments.
message RunPipelineArgs {
  // Required. The project in which to run the pipeline. The caller must have
  // WRITER access to all Google Cloud services and resources (e.g. Google
  // Compute Engine) will be used.
  string project_id = 1;

  // Pipeline input arguments; keys are defined in the pipeline documentation.
  // All input parameters that do not have default values  must be specified.
  // If parameters with defaults are specified here, the defaults will be
  // overridden.
  map<string, string> inputs = 2;

  // Pipeline output arguments; keys are defined in the pipeline
  // documentation.  All output parameters of without default values
  // must be specified.  If parameters with defaults are specified
  // here, the defaults will be overridden.
  map<string, string> outputs = 3;

  // The Google Cloud Service Account that will be used to access data and
  // services. By default, the compute service account associated with
  // `projectId` is used.
  ServiceAccount service_account = 4;

  // This field is deprecated. Use `labels` instead. Client-specified pipeline
  // operation identifier.
  string client_id = 5;

  // Specifies resource requirements/overrides for the pipeline run.
  PipelineResources resources = 6;

  // Required. Logging options. Used by the service to communicate results
  // to the user.
  LoggingOptions logging = 7;

  // How long to keep the VM up after a failure (for example docker command
  // failed, copying input or output files failed, etc). While the VM is up, one
  // can ssh into the VM to debug. Default is 0; maximum allowed value is 1 day.
  google.protobuf.Duration keep_vm_alive_on_failure_duration = 8;

  // Labels to apply to this pipeline run. Labels will also be applied to
  // compute resources (VM, disks) created by this pipeline run. When listing
  // operations, operations can [filtered by labels]
  // [google.longrunning.ListOperationsRequest.filter].
  // Label keys may not be empty; label values may be empty. Non-empty labels
  // must be 1-63 characters long, and comply with [RFC1035]
  // (https://www.ietf.org/rfc/rfc1035.txt).
  // Specifically, the name must be 1-63 characters long and match the regular
  // expression `[a-z]([-a-z0-9]*[a-z0-9])?` which means the first
  // character must be a lowercase letter, and all following characters must be
  // a dash, lowercase letter, or digit, except the last character, which cannot
  // be a dash.
  map<string, string> labels = 9;
}

// The request to run a pipeline. If `pipelineId` is specified, it
// refers to a saved pipeline created with CreatePipeline and set as
// the `pipelineId` of the returned Pipeline object. If
// `ephemeralPipeline` is specified, that pipeline is run once
// with the given args and not saved. It is an error to specify both
// `pipelineId` and `ephemeralPipeline`. `pipelineArgs`
// must be specified.
message RunPipelineRequest {
  oneof pipeline {
    // The already created pipeline to run.
    string pipeline_id = 1;

    // A new pipeline object to run once and then delete.
    Pipeline ephemeral_pipeline = 2;
  }

  // The arguments to use when running this pipeline.
  RunPipelineArgs pipeline_args = 3;
}

// A request to get a saved pipeline by id.
message GetPipelineRequest {
  // Caller must have READ access to the project in which this pipeline
  // is defined.
  string pipeline_id = 1;
}

// A request to list pipelines in a given project. Pipelines can be
// filtered by name using `namePrefix`: all pipelines with names that
// begin with `namePrefix` will be returned. Uses standard pagination:
// `pageSize` indicates how many pipelines to return, and
// `pageToken` comes from a previous ListPipelinesResponse to
// indicate offset.
message ListPipelinesRequest {
  // Required. The name of the project to search for pipelines. Caller
  // must have READ access to this project.
  string project_id = 1;

  // Pipelines with names that match this prefix should be
  // returned.  If unspecified, all pipelines in the project, up to
  // `pageSize`, will be returned.
  string name_prefix = 2;

  // Number of pipelines to return at once. Defaults to 256, and max
  // is 2048.
  int32 page_size = 3;

  // Token to use to indicate where to start getting results.
  // If unspecified, returns the first page of results.
  string page_token = 4;
}

// The response of ListPipelines. Contains at most `pageSize`
// pipelines. If it contains `pageSize` pipelines, and more pipelines
// exist, then `nextPageToken` will be populated and should be
// used as the `pageToken` argument to a subsequent ListPipelines
// request.
message ListPipelinesResponse {
  // The matched pipelines.
  repeated Pipeline pipelines = 1;

  // The token to use to get the next page of results.
  string next_page_token = 2;
}

// The request to delete a saved pipeline by ID.
message DeletePipelineRequest {
  // Caller must have WRITE access to the project in which this pipeline
  // is defined.
  string pipeline_id = 1;
}

// Request to get controller configuation.  Should only be used
// by VMs created by the Pipelines Service and not by end users.
message GetControllerConfigRequest {
  // The operation to retrieve controller configuration for.
  string operation_id = 1;

  uint64 validation_token = 2;
}

// Stores the information that the controller will fetch from the
// server in order to run. Should only be used by VMs created by the
// Pipelines Service and not by end users.
message ControllerConfig {
  message RepeatedString {
    repeated string values = 1;
  }

  string image = 1;

  string cmd = 2;

  string gcs_log_path = 3;

  string machine_type = 4;

  map<string, string> vars = 5;

  map<string, string> disks = 6;

  map<string, RepeatedString> gcs_sources = 7;

  map<string, RepeatedString> gcs_sinks = 8;
}

// Stores the list of events and times they occured for major events in job
// execution.
message TimestampEvent {
  // String indicating the type of event
  string description = 1;

  // The time this event occured.
  google.protobuf.Timestamp timestamp = 2;
}

// Request to set operation status. Should only be used by VMs
// created by the Pipelines Service and not by end users.
message SetOperationStatusRequest {
  string operation_id = 1;

  repeated TimestampEvent timestamp_events = 2;

  google.rpc.Code error_code = 3;

  string error_message = 4;

  uint64 validation_token = 5;
}

// A Google Cloud Service Account.
message ServiceAccount {
  // Email address of the service account. Defaults to `default`,
  // which uses the compute service account associated with the project.
  string email = 1;

  // List of scopes to be enabled for this service account on the VM.
  // The following scopes are automatically included:
  //
  // * https://www.googleapis.com/auth/compute
  // * https://www.googleapis.com/auth/devstorage.full_control
  // * https://www.googleapis.com/auth/genomics
  // * https://www.googleapis.com/auth/logging.write
  // * https://www.googleapis.com/auth/monitoring.write
  repeated string scopes = 2;
}

// The logging options for the pipeline run.
message LoggingOptions {
  // The location in Google Cloud Storage to which the pipeline logs
  // will be copied. Can be specified as a fully qualified directory
  // path, in which case logs will be output with a unique identifier
  // as the filename in that directory, or as a fully specified path,
  // which must end in `.log`, in which case that path will be
  // used, and the user must ensure that logs are not
  // overwritten. Stdout and stderr logs from the run are also
  // generated and output as `-stdout.log` and `-stderr.log`.
  string gcs_path = 1;
}

// The system resources for the pipeline run.
message PipelineResources {
  // A Google Compute Engine disk resource specification.
  message Disk {
    // The types of disks that may be attached to VMs.
    enum Type {
      // Default disk type. Use one of the other options below.
      TYPE_UNSPECIFIED = 0;

      // Specifies a Google Compute Engine persistent hard disk. See
      // https://cloud.google.com/compute/docs/disks/#pdspecs for details.
      PERSISTENT_HDD = 1;

      // Specifies a Google Compute Engine persistent solid-state disk. See
      // https://cloud.google.com/compute/docs/disks/#pdspecs for details.
      PERSISTENT_SSD = 2;

      // Specifies a Google Compute Engine local SSD.
      // See https://cloud.google.com/compute/docs/disks/local-ssd for details.
      LOCAL_SSD = 3;
    }

    // Required. The name of the disk that can be used in the pipeline
    // parameters. Must be 1 - 63 characters.
    // The name "boot" is reserved for system use.
    string name = 1;

    // Required. The type of the disk to create.
    Type type = 2;

    // The size of the disk. Defaults to 500 (GB).
    // This field is not applicable for local SSD.
    int32 size_gb = 3;

    // The full or partial URL of the persistent disk to attach. See
    // https://cloud.google.com/compute/docs/reference/latest/instances#resource
    // and
    // https://cloud.google.com/compute/docs/disks/persistent-disks#snapshots
    // for more details.
    string source = 4;

    // Deprecated. Disks created by the Pipelines API will be deleted at the end
    // of the pipeline run, regardless of what this field is set to.
    bool auto_delete = 6;

    // Required at create time and cannot be overridden at run time.
    // Specifies the path in the docker container where files on
    // this disk should be located. For example, if `mountPoint`
    // is `/mnt/disk`, and the parameter has `localPath`
    // `inputs/file.txt`, the docker container can access the data at
    // `/mnt/disk/inputs/file.txt`.
    string mount_point = 8;
  }

  // The minimum number of cores to use. Defaults to 1.
  int32 minimum_cpu_cores = 1;

  // Whether to use preemptible VMs. Defaults to `false`. In order to use this,
  // must be true for both create time and run time. Cannot be true at run time
  // if false at create time.
  bool preemptible = 2;

  // The minimum amount of RAM to use. Defaults to 3.75 (GB)
  double minimum_ram_gb = 3;

  // Disks to attach.
  repeated Disk disks = 4;

  // List of Google Compute Engine availability zones to which resource
  // creation will restricted. If empty, any zone may be chosen.
  repeated string zones = 5;

  // The size of the boot disk. Defaults to 10 (GB).
  int32 boot_disk_size_gb = 6;

  // Whether to assign an external IP to the instance. This is an experimental
  // feature that may go away. Defaults to false.
  // Corresponds to `--no_address` flag for [gcloud compute instances create]
  // (https://cloud.google.com/sdk/gcloud/reference/compute/instances/create).
  // In order to use this, must be true for both create time and run time.
  // Cannot be true at run time if false at create time. If you need to ssh into
  // a private IP VM for debugging, you can ssh to a public VM and then ssh into
  // the private VM's Internal IP.  If noAddress is set, this pipeline run may
  // only load docker images from Google Container Registry and not Docker Hub.
  // ** Note: To use this option, your project must be in Google Access for
  // Private IPs Early Access Program.**
  bool no_address = 7;
}

// Parameters facilitate setting and delivering data into the
// pipeline's execution environment. They are defined at create time,
// with optional defaults, and can be overridden at run time.
//
// If `localCopy` is unset, then the parameter specifies a string that
// is passed as-is into the pipeline, as the value of the environment
// variable with the given name.  A default value can be optionally
// specified at create time. The default can be overridden at run time
// using the inputs map. If no default is given, a value must be
// supplied at runtime.
//
// If `localCopy` is defined, then the parameter specifies a data
// source or sink, both in Google Cloud Storage and on the Docker container
// where the pipeline computation is run. The [service account associated with
// the Pipeline][google.genomics.v1alpha2.RunPipelineArgs.service_account] (by
// default the project's Compute Engine service account) must have access to the
// Google Cloud Storage paths.
//
// At run time, the Google Cloud Storage paths can be overridden if a default
// was provided at create time, or must be set otherwise. The pipeline runner
// should add a key/value pair to either the inputs or outputs map. The
// indicated data copies will be carried out before/after pipeline execution,
// just as if the corresponding arguments were provided to `gsutil cp`.
//
// For example: Given the following `PipelineParameter`, specified
// in the `inputParameters` list:
//
// ```
// {name: "input_file", localCopy: {path: "file.txt", disk: "pd1"}}
// ```
//
// where `disk` is defined in the `PipelineResources` object as:
//
// ```
// {name: "pd1", mountPoint: "/mnt/disk/"}
// ```
//
// We create a disk named `pd1`, mount it on the host VM, and map
// `/mnt/pd1` to `/mnt/disk` in the docker container.  At
// runtime, an entry for `input_file` would be required in the inputs
// map, such as:
//
// ```
//   inputs["input_file"] = "gs://my-bucket/bar.txt"
// ```
//
// This would generate the following gsutil call:
//
// ```
//   gsutil cp gs://my-bucket/bar.txt /mnt/pd1/file.txt
// ```
//
// The file `/mnt/pd1/file.txt` maps to `/mnt/disk/file.txt` in the
// Docker container. Acceptable paths are:
//
// <table>
//   <thead>
//     <tr><th>Google Cloud storage path</th><th>Local path</th></tr>
//   </thead>
//   <tbody>
//     <tr><td>file</td><td>file</td></tr>
//     <tr><td>glob</td><td>directory</td></tr>
//   </tbody>
// </table>
//
// For outputs, the direction of the copy is reversed:
//
// ```
//   gsutil cp /mnt/disk/file.txt gs://my-bucket/bar.txt
// ```
//
// Acceptable paths are:
//
// <table>
//   <thead>
//     <tr><th>Local path</th><th>Google Cloud Storage path</th></tr>
//   </thead>
//   <tbody>
//     <tr><td>file</td><td>file</td></tr>
//     <tr>
//       <td>file</td>
//       <td>directory - directory must already exist</td>
//     </tr>
//     <tr>
//       <td>glob</td>
//       <td>directory - directory will be created if it doesn't exist</td></tr>
//   </tbody>
// </table>
//
// One restriction due to docker limitations, is that for outputs that are found
// on the boot disk, the local path cannot be a glob and must be a file.
message PipelineParameter {
  // LocalCopy defines how a remote file should be copied to and from the VM.
  message LocalCopy {
    // Required. The path within the user's docker container where
    // this input should be localized to and from, relative to the specified
    // disk's mount point. For example: file.txt,
    string path = 1;

    // Required. The name of the disk where this parameter is
    // located. Can be the name of one of the disks specified in the
    // Resources field, or "boot", which represents the Docker
    // instance's boot disk and has a mount point of `/`.
    string disk = 2;
  }

  // Required. Name of the parameter - the pipeline runner uses this string
  // as the key to the input and output maps in RunPipeline.
  string name = 1;

  // Human-readable description.
  string description = 2;

  // The default value for this parameter. Can be overridden at runtime.
  // If `localCopy` is present, then this must be a Google Cloud Storage path
  // beginning with `gs://`.
  string default_value = 5;

  // If present, this parameter is marked for copying to and from the VM.
  // `LocalCopy` indicates where on the VM the file should be. The value
  // given to this parameter (either at runtime or using `defaultValue`)
  // must be the remote path where the file should be.
  LocalCopy local_copy = 6;
}

// The Docker execuctor specification.
message DockerExecutor {
  // Required. Image name from either Docker Hub or Google Container Registry.
  // Users that run pipelines must have READ access to the image.
  string image_name = 1;

  // Required. The command or newline delimited script to run. The command
  // string will be executed within a bash shell.
  //
  // If the command exits with a non-zero exit code, output parameter
  // de-localization will be skipped and the pipeline operation's
  // [`error`][google.longrunning.Operation.error] field will be populated.
  //
  // Maximum command string length is 16384.
  string cmd = 2;
}