resources.proto 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790
  1. // Copyright 2022 Google LLC
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. syntax = "proto3";
  15. package google.cloud.dataplex.v1;
  16. import "google/api/field_behavior.proto";
  17. import "google/api/resource.proto";
  18. import "google/protobuf/duration.proto";
  19. import "google/protobuf/timestamp.proto";
  20. option go_package = "google.golang.org/genproto/googleapis/cloud/dataplex/v1;dataplex";
  21. option java_multiple_files = true;
  22. option java_outer_classname = "ResourcesProto";
  23. option java_package = "com.google.cloud.dataplex.v1";
  24. // A lake is a centralized repository for managing enterprise data across the
  25. // organization distributed across many cloud projects, and stored in a variety
  26. // of storage services such as Google Cloud Storage and BigQuery. The resources
  27. // attached to a lake are referred to as managed resources. Data within these
  28. // managed resources can be structured or unstructured. A lake provides data
  29. // admins with tools to organize, secure and manage their data at scale, and
  30. // provides data scientists and data engineers an integrated experience to
  31. // easily search, discover, analyze and transform data and associated metadata.
  32. message Lake {
  33. option (google.api.resource) = {
  34. type: "dataplex.googleapis.com/Lake"
  35. pattern: "projects/{project}/locations/{location}/lakes/{lake}"
  36. };
  37. // Settings to manage association of Dataproc Metastore with a lake.
  38. message Metastore {
  39. // Optional. A relative reference to the Dataproc Metastore
  40. // (https://cloud.google.com/dataproc-metastore/docs) service associated
  41. // with the lake:
  42. // `projects/{project_id}/locations/{location_id}/services/{service_id}`
  43. string service = 1 [(google.api.field_behavior) = OPTIONAL];
  44. }
  45. // Status of Lake and Dataproc Metastore service instance association.
  46. message MetastoreStatus {
  47. // Current state of association.
  48. enum State {
  49. // Unspecified.
  50. STATE_UNSPECIFIED = 0;
  51. // A Metastore service instance is not associated with the lake.
  52. NONE = 1;
  53. // A Metastore service instance is attached to the lake.
  54. READY = 2;
  55. // Attach/detach is in progress.
  56. UPDATING = 3;
  57. // Attach/detach could not be done due to errors.
  58. ERROR = 4;
  59. }
  60. // Current state of association.
  61. State state = 1;
  62. // Additional information about the current status.
  63. string message = 2;
  64. // Last update time of the metastore status of the lake.
  65. google.protobuf.Timestamp update_time = 3;
  66. // The URI of the endpoint used to access the Metastore service.
  67. string endpoint = 4;
  68. }
  69. // Output only. The relative resource name of the lake, of the form:
  70. // `projects/{project_number}/locations/{location_id}/lakes/{lake_id}`.
  71. string name = 1 [
  72. (google.api.field_behavior) = OUTPUT_ONLY,
  73. (google.api.resource_reference) = {
  74. type: "dataplex.googleapis.com/Lake"
  75. }
  76. ];
  77. // Optional. User friendly display name.
  78. string display_name = 2 [(google.api.field_behavior) = OPTIONAL];
  79. // Output only. System generated globally unique ID for the lake. This ID will be
  80. // different if the lake is deleted and re-created with the same name.
  81. string uid = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
  82. // Output only. The time when the lake was created.
  83. google.protobuf.Timestamp create_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
  84. // Output only. The time when the lake was last updated.
  85. google.protobuf.Timestamp update_time = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
  86. // Optional. User-defined labels for the lake.
  87. map<string, string> labels = 6 [(google.api.field_behavior) = OPTIONAL];
  88. // Optional. Description of the lake.
  89. string description = 7 [(google.api.field_behavior) = OPTIONAL];
  90. // Output only. Current state of the lake.
  91. State state = 8 [(google.api.field_behavior) = OUTPUT_ONLY];
  92. // Output only. Service account associated with this lake. This service account must be
  93. // authorized to access or operate on resources managed by the lake.
  94. string service_account = 9 [(google.api.field_behavior) = OUTPUT_ONLY];
  95. // Optional. Settings to manage lake and Dataproc Metastore service instance
  96. // association.
  97. Metastore metastore = 102 [(google.api.field_behavior) = OPTIONAL];
  98. // Output only. Aggregated status of the underlying assets of the lake.
  99. AssetStatus asset_status = 103 [(google.api.field_behavior) = OUTPUT_ONLY];
  100. // Output only. Metastore status of the lake.
  101. MetastoreStatus metastore_status = 104 [(google.api.field_behavior) = OUTPUT_ONLY];
  102. }
  103. // Aggregated status of the underlying assets of a lake or zone.
  104. message AssetStatus {
  105. // Last update time of the status.
  106. google.protobuf.Timestamp update_time = 1;
  107. // Number of active assets.
  108. int32 active_assets = 2;
  109. // Number of assets that are in process of updating the security policy on
  110. // attached resources.
  111. int32 security_policy_applying_assets = 3;
  112. }
  113. // A zone represents a logical group of related assets within a lake. A zone can
  114. // be used to map to organizational structure or represent stages of data
  115. // readiness from raw to curated. It provides managing behavior that is shared
  116. // or inherited by all contained assets.
  117. message Zone {
  118. option (google.api.resource) = {
  119. type: "dataplex.googleapis.com/Zone"
  120. pattern: "projects/{project}/locations/{location}/lakes/{lake}/zones/{zone}"
  121. };
  122. // Type of zone.
  123. enum Type {
  124. // Zone type not specified.
  125. TYPE_UNSPECIFIED = 0;
  126. // A zone that contains data that needs further processing before it is
  127. // considered generally ready for consumption and analytics workloads.
  128. RAW = 1;
  129. // A zone that contains data that is considered to be ready for broader
  130. // consumption and analytics workloads. Curated structured data stored in
  131. // Cloud Storage must conform to certain file formats (parquet, avro and
  132. // orc) and organized in a hive-compatible directory layout.
  133. CURATED = 2;
  134. }
  135. // Settings for resources attached as assets within a zone.
  136. message ResourceSpec {
  137. // Location type of the resources attached to a zone.
  138. enum LocationType {
  139. // Unspecified location type.
  140. LOCATION_TYPE_UNSPECIFIED = 0;
  141. // Resources that are associated with a single region.
  142. SINGLE_REGION = 1;
  143. // Resources that are associated with a multi-region location.
  144. MULTI_REGION = 2;
  145. }
  146. // Required. Immutable. The location type of the resources that are allowed to be attached to the
  147. // assets within this zone.
  148. LocationType location_type = 1 [
  149. (google.api.field_behavior) = REQUIRED,
  150. (google.api.field_behavior) = IMMUTABLE
  151. ];
  152. }
  153. // Settings to manage the metadata discovery and publishing in a zone.
  154. message DiscoverySpec {
  155. // Describe CSV and similar semi-structured data formats.
  156. message CsvOptions {
  157. // Optional. The number of rows to interpret as header rows that should be skipped
  158. // when reading data rows.
  159. int32 header_rows = 1 [(google.api.field_behavior) = OPTIONAL];
  160. // Optional. The delimiter being used to separate values. This defaults to ','.
  161. string delimiter = 2 [(google.api.field_behavior) = OPTIONAL];
  162. // Optional. The character encoding of the data. The default is UTF-8.
  163. string encoding = 3 [(google.api.field_behavior) = OPTIONAL];
  164. // Optional. Whether to disable the inference of data type for CSV data.
  165. // If true, all columns will be registered as strings.
  166. bool disable_type_inference = 4 [(google.api.field_behavior) = OPTIONAL];
  167. }
  168. // Describe JSON data format.
  169. message JsonOptions {
  170. // Optional. The character encoding of the data. The default is UTF-8.
  171. string encoding = 1 [(google.api.field_behavior) = OPTIONAL];
  172. // Optional. Whether to disable the inference of data type for Json data.
  173. // If true, all columns will be registered as their primitive types
  174. // (strings, number or boolean).
  175. bool disable_type_inference = 2 [(google.api.field_behavior) = OPTIONAL];
  176. }
  177. // Required. Whether discovery is enabled.
  178. bool enabled = 1 [(google.api.field_behavior) = REQUIRED];
  179. // Optional. The list of patterns to apply for selecting data to include during
  180. // discovery if only a subset of the data should considered. For Cloud
  181. // Storage bucket assets, these are interpreted as glob patterns used to
  182. // match object names. For BigQuery dataset assets, these are
  183. // interpreted as patterns to match table names.
  184. repeated string include_patterns = 2 [(google.api.field_behavior) = OPTIONAL];
  185. // Optional. The list of patterns to apply for selecting data to exclude during
  186. // discovery. For Cloud Storage bucket assets, these are interpreted as
  187. // glob patterns used to match object names. For BigQuery dataset assets,
  188. // these are interpreted as patterns to match table names.
  189. repeated string exclude_patterns = 3 [(google.api.field_behavior) = OPTIONAL];
  190. // Optional. Configuration for CSV data.
  191. CsvOptions csv_options = 4 [(google.api.field_behavior) = OPTIONAL];
  192. // Optional. Configuration for Json data.
  193. JsonOptions json_options = 5 [(google.api.field_behavior) = OPTIONAL];
  194. // Determines when discovery is triggered.
  195. oneof trigger {
  196. // Optional. Cron schedule (https://en.wikipedia.org/wiki/Cron) for running
  197. // discovery periodically. Successive discovery runs must be scheduled at
  198. // least 60 minutes apart.
  199. // The default value is to run discovery every 60 minutes.
  200. // To explicitly set a timezone to the cron tab, apply a prefix in the
  201. // cron tab: "CRON_TZ=${IANA_TIME_ZONE}" or TZ=${IANA_TIME_ZONE}".
  202. // The ${IANA_TIME_ZONE} may only be a valid string from IANA time zone
  203. // database. For example, "CRON_TZ=America/New_York 1 * * * *", or
  204. // "TZ=America/New_York 1 * * * *".
  205. string schedule = 10 [(google.api.field_behavior) = OPTIONAL];
  206. }
  207. }
  208. // Output only. The relative resource name of the zone, of the form:
  209. // `projects/{project_number}/locations/{location_id}/lakes/{lake_id}/zones/{zone_id}`.
  210. string name = 1 [
  211. (google.api.field_behavior) = OUTPUT_ONLY,
  212. (google.api.resource_reference) = {
  213. type: "dataplex.googleapis.com/Zone"
  214. }
  215. ];
  216. // Optional. User friendly display name.
  217. string display_name = 2 [(google.api.field_behavior) = OPTIONAL];
  218. // Output only. System generated globally unique ID for the zone. This ID will be
  219. // different if the zone is deleted and re-created with the same name.
  220. string uid = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
  221. // Output only. The time when the zone was created.
  222. google.protobuf.Timestamp create_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
  223. // Output only. The time when the zone was last updated.
  224. google.protobuf.Timestamp update_time = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
  225. // Optional. User defined labels for the zone.
  226. map<string, string> labels = 6 [(google.api.field_behavior) = OPTIONAL];
  227. // Optional. Description of the zone.
  228. string description = 7 [(google.api.field_behavior) = OPTIONAL];
  229. // Output only. Current state of the zone.
  230. State state = 8 [(google.api.field_behavior) = OUTPUT_ONLY];
  231. // Required. Immutable. The type of the zone.
  232. Type type = 9 [
  233. (google.api.field_behavior) = REQUIRED,
  234. (google.api.field_behavior) = IMMUTABLE
  235. ];
  236. // Optional. Specification of the discovery feature applied to data in this zone.
  237. DiscoverySpec discovery_spec = 103 [(google.api.field_behavior) = OPTIONAL];
  238. // Required. Specification of the resources that are referenced by the assets within
  239. // this zone.
  240. ResourceSpec resource_spec = 104 [(google.api.field_behavior) = REQUIRED];
  241. // Output only. Aggregated status of the underlying assets of the zone.
  242. AssetStatus asset_status = 105 [(google.api.field_behavior) = OUTPUT_ONLY];
  243. }
  244. // Action represents an issue requiring administrator action for resolution.
  245. message Action {
  246. option (google.api.resource) = {
  247. type: "dataplex.googleapis.com/Action"
  248. pattern: "projects/{project}/locations/{location}/lakes/{lake}/actions/{action}"
  249. pattern: "projects/{project}/locations/{location}/lakes/{lake}/zones/{zone}/actions/{action}"
  250. pattern: "projects/{project}/locations/{location}/lakes/{lake}/zones/{zone}/assets/{asset}/actions/{action}"
  251. };
  252. // The category of issues.
  253. enum Category {
  254. // Unspecified category.
  255. CATEGORY_UNSPECIFIED = 0;
  256. // Resource management related issues.
  257. RESOURCE_MANAGEMENT = 1;
  258. // Security policy related issues.
  259. SECURITY_POLICY = 2;
  260. // Data and discovery related issues.
  261. DATA_DISCOVERY = 3;
  262. }
  263. // Action details for resource references in assets that cannot be located.
  264. message MissingResource {
  265. }
  266. // Action details for unauthorized resource issues raised to indicate that the
  267. // service account associated with the lake instance is not authorized to
  268. // access or manage the resource associated with an asset.
  269. message UnauthorizedResource {
  270. }
  271. // Failed to apply security policy to the managed resource(s) under a
  272. // lake, zone or an asset. For a lake or zone resource, one or more underlying
  273. // assets has a failure applying security policy to the associated managed
  274. // resource.
  275. message FailedSecurityPolicyApply {
  276. // Resource name of one of the assets with failing security policy
  277. // application. Populated for a lake or zone resource only.
  278. string asset = 1;
  279. }
  280. // Action details for invalid or unsupported data files detected by discovery.
  281. message InvalidDataFormat {
  282. // The list of data locations sampled and used for format/schema
  283. // inference.
  284. repeated string sampled_data_locations = 1;
  285. // The expected data format of the entity.
  286. string expected_format = 2;
  287. // The new unexpected data format within the entity.
  288. string new_format = 3;
  289. }
  290. // Action details for incompatible schemas detected by discovery.
  291. message IncompatibleDataSchema {
  292. // Whether the action relates to a schema that is incompatible or modified.
  293. enum SchemaChange {
  294. // Schema change unspecified.
  295. SCHEMA_CHANGE_UNSPECIFIED = 0;
  296. // Newly discovered schema is incompatible with existing schema.
  297. INCOMPATIBLE = 1;
  298. // Newly discovered schema has changed from existing schema for data in a
  299. // curated zone.
  300. MODIFIED = 2;
  301. }
  302. // The name of the table containing invalid data.
  303. string table = 1;
  304. // The existing and expected schema of the table. The schema is provided as
  305. // a JSON formatted structure listing columns and data types.
  306. string existing_schema = 2;
  307. // The new and incompatible schema within the table. The schema is provided
  308. // as a JSON formatted structured listing columns and data types.
  309. string new_schema = 3;
  310. // The list of data locations sampled and used for format/schema
  311. // inference.
  312. repeated string sampled_data_locations = 4;
  313. // Whether the action relates to a schema that is incompatible or modified.
  314. SchemaChange schema_change = 5;
  315. }
  316. // Action details for invalid or unsupported partitions detected by discovery.
  317. message InvalidDataPartition {
  318. // The expected partition structure.
  319. enum PartitionStructure {
  320. // PartitionStructure unspecified.
  321. PARTITION_STRUCTURE_UNSPECIFIED = 0;
  322. // Consistent hive-style partition definition (both raw and curated zone).
  323. CONSISTENT_KEYS = 1;
  324. // Hive style partition definition (curated zone only).
  325. HIVE_STYLE_KEYS = 2;
  326. }
  327. // The issue type of InvalidDataPartition.
  328. PartitionStructure expected_structure = 1;
  329. }
  330. // Action details for absence of data detected by discovery.
  331. message MissingData {
  332. }
  333. // Action details for invalid data arrangement.
  334. message InvalidDataOrganization {
  335. }
  336. // The category of issue associated with the action.
  337. Category category = 1;
  338. // Detailed description of the issue requiring action.
  339. string issue = 2;
  340. // The time that the issue was detected.
  341. google.protobuf.Timestamp detect_time = 4;
  342. // Output only. The relative resource name of the action, of the form:
  343. // `projects/{project}/locations/{location}/lakes/{lake}/actions/{action}`
  344. // `projects/{project}/locations/{location}/lakes/{lake}/zones/{zone}/actions/{action}`
  345. // `projects/{project}/locations/{location}/lakes/{lake}/zones/{zone}/assets/{asset}/actions/{action}`.
  346. string name = 5 [
  347. (google.api.field_behavior) = OUTPUT_ONLY,
  348. (google.api.resource_reference) = {
  349. type: "dataplex.googleapis.com/Action"
  350. }
  351. ];
  352. // Output only. The relative resource name of the lake, of the form:
  353. // `projects/{project_number}/locations/{location_id}/lakes/{lake_id}`.
  354. string lake = 6 [
  355. (google.api.field_behavior) = OUTPUT_ONLY,
  356. (google.api.resource_reference) = {
  357. type: "dataplex.googleapis.com/Lake"
  358. }
  359. ];
  360. // Output only. The relative resource name of the zone, of the form:
  361. // `projects/{project_number}/locations/{location_id}/lakes/{lake_id}/zones/{zone_id}`.
  362. string zone = 7 [
  363. (google.api.field_behavior) = OUTPUT_ONLY,
  364. (google.api.resource_reference) = {
  365. type: "dataplex.googleapis.com/Zone"
  366. }
  367. ];
  368. // Output only. The relative resource name of the asset, of the form:
  369. // `projects/{project_number}/locations/{location_id}/lakes/{lake_id}/zones/{zone_id}/assets/{asset_id}`.
  370. string asset = 8 [
  371. (google.api.field_behavior) = OUTPUT_ONLY,
  372. (google.api.resource_reference) = {
  373. type: "dataplex.googleapis.com/Asset"
  374. }
  375. ];
  376. // The list of data locations associated with this action. Cloud Storage
  377. // locations are represented as URI paths(E.g.
  378. // `gs://bucket/table1/year=2020/month=Jan/`). BigQuery locations refer to
  379. // resource names(E.g.
  380. // `bigquery.googleapis.com/projects/project-id/datasets/dataset-id`).
  381. repeated string data_locations = 9;
  382. // Additional details about the action based on the action category.
  383. oneof details {
  384. // Details for issues related to invalid or unsupported data formats.
  385. InvalidDataFormat invalid_data_format = 10;
  386. // Details for issues related to incompatible schemas detected within data.
  387. IncompatibleDataSchema incompatible_data_schema = 11;
  388. // Details for issues related to invalid or unsupported data partition
  389. // structure.
  390. InvalidDataPartition invalid_data_partition = 12;
  391. // Details for issues related to absence of data within managed resources.
  392. MissingData missing_data = 13;
  393. // Details for issues related to absence of a managed resource.
  394. MissingResource missing_resource = 14;
  395. // Details for issues related to lack of permissions to access data
  396. // resources.
  397. UnauthorizedResource unauthorized_resource = 15;
  398. // Details for issues related to applying security policy.
  399. FailedSecurityPolicyApply failed_security_policy_apply = 21;
  400. // Details for issues related to invalid data arrangement.
  401. InvalidDataOrganization invalid_data_organization = 22;
  402. }
  403. }
  404. // An asset represents a cloud resource that is being managed within a lake as a
  405. // member of a zone.
  406. message Asset {
  407. option (google.api.resource) = {
  408. type: "dataplex.googleapis.com/Asset"
  409. pattern: "projects/{project}/locations/{location}/lakes/{lake}/zones/{zone}/assets/{asset}"
  410. };
  411. // Security policy status of the asset. Data security policy, i.e., readers,
  412. // writers & owners, should be specified in the lake/zone/asset IAM policy.
  413. message SecurityStatus {
  414. // The state of the security policy.
  415. enum State {
  416. // State unspecified.
  417. STATE_UNSPECIFIED = 0;
  418. // Security policy has been successfully applied to the attached resource.
  419. READY = 1;
  420. // Security policy is in the process of being applied to the attached
  421. // resource.
  422. APPLYING = 2;
  423. // Security policy could not be applied to the attached resource due to
  424. // errors.
  425. ERROR = 3;
  426. }
  427. // The current state of the security policy applied to the attached
  428. // resource.
  429. State state = 1;
  430. // Additional information about the current state.
  431. string message = 2;
  432. // Last update time of the status.
  433. google.protobuf.Timestamp update_time = 3;
  434. }
  435. // Settings to manage the metadata discovery and publishing for an asset.
  436. message DiscoverySpec {
  437. // Describe CSV and similar semi-structured data formats.
  438. message CsvOptions {
  439. // Optional. The number of rows to interpret as header rows that should be skipped
  440. // when reading data rows.
  441. int32 header_rows = 1 [(google.api.field_behavior) = OPTIONAL];
  442. // Optional. The delimiter being used to separate values. This defaults to ','.
  443. string delimiter = 2 [(google.api.field_behavior) = OPTIONAL];
  444. // Optional. The character encoding of the data. The default is UTF-8.
  445. string encoding = 3 [(google.api.field_behavior) = OPTIONAL];
  446. // Optional. Whether to disable the inference of data type for CSV data.
  447. // If true, all columns will be registered as strings.
  448. bool disable_type_inference = 4 [(google.api.field_behavior) = OPTIONAL];
  449. }
  450. // Describe JSON data format.
  451. message JsonOptions {
  452. // Optional. The character encoding of the data. The default is UTF-8.
  453. string encoding = 1 [(google.api.field_behavior) = OPTIONAL];
  454. // Optional. Whether to disable the inference of data type for Json data.
  455. // If true, all columns will be registered as their primitive types
  456. // (strings, number or boolean).
  457. bool disable_type_inference = 2 [(google.api.field_behavior) = OPTIONAL];
  458. }
  459. // Optional. Whether discovery is enabled.
  460. bool enabled = 1 [(google.api.field_behavior) = OPTIONAL];
  461. // Optional. The list of patterns to apply for selecting data to include during
  462. // discovery if only a subset of the data should considered. For Cloud
  463. // Storage bucket assets, these are interpreted as glob patterns used to
  464. // match object names. For BigQuery dataset assets, these are interpreted as
  465. // patterns to match table names.
  466. repeated string include_patterns = 2 [(google.api.field_behavior) = OPTIONAL];
  467. // Optional. The list of patterns to apply for selecting data to exclude during
  468. // discovery. For Cloud Storage bucket assets, these are interpreted as
  469. // glob patterns used to match object names. For BigQuery dataset assets,
  470. // these are interpreted as patterns to match table names.
  471. repeated string exclude_patterns = 3 [(google.api.field_behavior) = OPTIONAL];
  472. // Optional. Configuration for CSV data.
  473. CsvOptions csv_options = 4 [(google.api.field_behavior) = OPTIONAL];
  474. // Optional. Configuration for Json data.
  475. JsonOptions json_options = 5 [(google.api.field_behavior) = OPTIONAL];
  476. // Determines when discovery is triggered.
  477. oneof trigger {
  478. // Optional. Cron schedule (https://en.wikipedia.org/wiki/Cron) for running
  479. // discovery periodically. Successive discovery runs must be scheduled at
  480. // least 60 minutes apart.
  481. // The default value is to run discovery every 60 minutes.
  482. // To explicitly set a timezone to the cron tab, apply a prefix in the
  483. // cron tab: "CRON_TZ=${IANA_TIME_ZONE}" or TZ=${IANA_TIME_ZONE}".
  484. // The ${IANA_TIME_ZONE} may only be a valid string from IANA time zone
  485. // database. For example, "CRON_TZ=America/New_York 1 * * * *", or
  486. // "TZ=America/New_York 1 * * * *".
  487. string schedule = 10 [(google.api.field_behavior) = OPTIONAL];
  488. }
  489. }
  490. // Identifies the cloud resource that is referenced by this asset.
  491. message ResourceSpec {
  492. // Type of resource.
  493. enum Type {
  494. // Type not specified.
  495. TYPE_UNSPECIFIED = 0;
  496. // Cloud Storage bucket.
  497. STORAGE_BUCKET = 1;
  498. // BigQuery dataset.
  499. BIGQUERY_DATASET = 2;
  500. }
  501. // Immutable. Relative name of the cloud resource that contains the data that is
  502. // being managed within a lake. For example:
  503. // `projects/{project_number}/buckets/{bucket_id}`
  504. // `projects/{project_number}/datasets/{dataset_id}`
  505. string name = 1 [(google.api.field_behavior) = IMMUTABLE];
  506. // Required. Immutable. Type of resource.
  507. Type type = 2 [
  508. (google.api.field_behavior) = REQUIRED,
  509. (google.api.field_behavior) = IMMUTABLE
  510. ];
  511. }
  512. // Status of the resource referenced by an asset.
  513. message ResourceStatus {
  514. // The state of a resource.
  515. enum State {
  516. // State unspecified.
  517. STATE_UNSPECIFIED = 0;
  518. // Resource does not have any errors.
  519. READY = 1;
  520. // Resource has errors.
  521. ERROR = 2;
  522. }
  523. // The current state of the managed resource.
  524. State state = 1;
  525. // Additional information about the current state.
  526. string message = 2;
  527. // Last update time of the status.
  528. google.protobuf.Timestamp update_time = 3;
  529. }
  530. // Status of discovery for an asset.
  531. message DiscoveryStatus {
  532. // Current state of discovery.
  533. enum State {
  534. // State is unspecified.
  535. STATE_UNSPECIFIED = 0;
  536. // Discovery for the asset is scheduled.
  537. SCHEDULED = 1;
  538. // Discovery for the asset is running.
  539. IN_PROGRESS = 2;
  540. // Discovery for the asset is currently paused (e.g. due to a lack
  541. // of available resources). It will be automatically resumed.
  542. PAUSED = 3;
  543. // Discovery for the asset is disabled.
  544. DISABLED = 5;
  545. }
  546. // The aggregated data statistics for the asset reported by discovery.
  547. message Stats {
  548. // The count of data items within the referenced resource.
  549. int64 data_items = 1;
  550. // The number of stored data bytes within the referenced resource.
  551. int64 data_size = 2;
  552. // The count of table entities within the referenced resource.
  553. int64 tables = 3;
  554. // The count of fileset entities within the referenced resource.
  555. int64 filesets = 4;
  556. }
  557. // The current status of the discovery feature.
  558. State state = 1;
  559. // Additional information about the current state.
  560. string message = 2;
  561. // Last update time of the status.
  562. google.protobuf.Timestamp update_time = 3;
  563. // The start time of the last discovery run.
  564. google.protobuf.Timestamp last_run_time = 4;
  565. // Data Stats of the asset reported by discovery.
  566. Stats stats = 6;
  567. // The duration of the last discovery run.
  568. google.protobuf.Duration last_run_duration = 7;
  569. }
  570. // Output only. The relative resource name of the asset, of the form:
  571. // `projects/{project_number}/locations/{location_id}/lakes/{lake_id}/zones/{zone_id}/assets/{asset_id}`.
  572. string name = 1 [
  573. (google.api.field_behavior) = OUTPUT_ONLY,
  574. (google.api.resource_reference) = {
  575. type: "dataplex.googleapis.com/Asset"
  576. }
  577. ];
  578. // Optional. User friendly display name.
  579. string display_name = 2 [(google.api.field_behavior) = OPTIONAL];
  580. // Output only. System generated globally unique ID for the asset. This ID will be
  581. // different if the asset is deleted and re-created with the same name.
  582. string uid = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
  583. // Output only. The time when the asset was created.
  584. google.protobuf.Timestamp create_time = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
  585. // Output only. The time when the asset was last updated.
  586. google.protobuf.Timestamp update_time = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
  587. // Optional. User defined labels for the asset.
  588. map<string, string> labels = 6 [(google.api.field_behavior) = OPTIONAL];
  589. // Optional. Description of the asset.
  590. string description = 7 [(google.api.field_behavior) = OPTIONAL];
  591. // Output only. Current state of the asset.
  592. State state = 8 [(google.api.field_behavior) = OUTPUT_ONLY];
  593. // Required. Specification of the resource that is referenced by this asset.
  594. ResourceSpec resource_spec = 100 [(google.api.field_behavior) = REQUIRED];
  595. // Output only. Status of the resource referenced by this asset.
  596. ResourceStatus resource_status = 101 [(google.api.field_behavior) = OUTPUT_ONLY];
  597. // Output only. Status of the security policy applied to resource referenced by this asset.
  598. SecurityStatus security_status = 103 [(google.api.field_behavior) = OUTPUT_ONLY];
  599. // Optional. Specification of the discovery feature applied to data referenced by this
  600. // asset.
  601. // When this spec is left unset, the asset will use the spec set on the parent
  602. // zone.
  603. DiscoverySpec discovery_spec = 106 [(google.api.field_behavior) = OPTIONAL];
  604. // Output only. Status of the discovery feature applied to data referenced by this asset.
  605. DiscoveryStatus discovery_status = 107 [(google.api.field_behavior) = OUTPUT_ONLY];
  606. }
  607. // State of a resource.
  608. enum State {
  609. // State is not specified.
  610. STATE_UNSPECIFIED = 0;
  611. // Resource is active, i.e., ready to use.
  612. ACTIVE = 1;
  613. // Resource is under creation.
  614. CREATING = 2;
  615. // Resource is under deletion.
  616. DELETING = 3;
  617. // Resource is active but has unresolved actions.
  618. ACTION_REQUIRED = 4;
  619. }