databricks.getDataQualityMonitor
This data source can be used to fetch a data quality monitor.
For the table object_type, the caller must either:
- be an owner of the table’s parent catalog
- have USE_CATALOG on the table’s parent catalog and be an owner of the table’s parent schema.
- have the following permissions:
- USE_CATALOG on the table’s parent catalog
- USE_SCHEMA on the table’s parent schema
- SELECT privilege on the table.
Note This data source can only be used with a workspace-level provider!
Example Usage
Getting a data quality monitor by Unity Catalog object type (currently supports schema and table) and object id:
import * as pulumi from "@pulumi/pulumi";
import * as databricks from "@pulumi/databricks";
const _this = databricks.getSchema({
name: "my_catalog.my_schema",
});
const thisGetDataQualityMonitor = _this.then(_this => databricks.getDataQualityMonitor({
objectType: "schema",
objectId: _this.schemaInfo?.schemaId,
}));
import pulumi
import pulumi_databricks as databricks
this = databricks.get_schema(name="my_catalog.my_schema")
this_get_data_quality_monitor = databricks.get_data_quality_monitor(object_type="schema",
object_id=this.schema_info.schema_id)
package main
import (
"github.com/pulumi/pulumi-databricks/sdk/go/databricks"
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)
func main() {
pulumi.Run(func(ctx *pulumi.Context) error {
this, err := databricks.LookupSchema(ctx, &databricks.LookupSchemaArgs{
Name: "my_catalog.my_schema",
}, nil)
if err != nil {
return err
}
_, err = databricks.LookupDataQualityMonitor(ctx, &databricks.LookupDataQualityMonitorArgs{
ObjectType: "schema",
ObjectId: this.SchemaInfo.SchemaId,
}, nil)
if err != nil {
return err
}
return nil
})
}
using System.Collections.Generic;
using System.Linq;
using Pulumi;
using Databricks = Pulumi.Databricks;
return await Deployment.RunAsync(() =>
{
var @this = Databricks.GetSchema.Invoke(new()
{
Name = "my_catalog.my_schema",
});
var thisGetDataQualityMonitor = Databricks.GetDataQualityMonitor.Invoke(new()
{
ObjectType = "schema",
ObjectId = @this.Apply(getSchemaResult => getSchemaResult.SchemaInfo?.SchemaId),
});
});
package generated_program;
import com.pulumi.Context;
import com.pulumi.Pulumi;
import com.pulumi.core.Output;
import com.pulumi.databricks.DatabricksFunctions;
import com.pulumi.databricks.inputs.GetSchemaArgs;
import com.pulumi.databricks.inputs.GetDataQualityMonitorArgs;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Paths;
public class App {
public static void main(String[] args) {
Pulumi.run(App::stack);
}
public static void stack(Context ctx) {
final var this = DatabricksFunctions.getSchema(GetSchemaArgs.builder()
.name("my_catalog.my_schema")
.build());
final var thisGetDataQualityMonitor = DatabricksFunctions.getDataQualityMonitor(GetDataQualityMonitorArgs.builder()
.objectType("schema")
.objectId(this_.schemaInfo().schemaId())
.build());
}
}
variables:
this:
fn::invoke:
function: databricks:getSchema
arguments:
name: my_catalog.my_schema
thisGetDataQualityMonitor:
fn::invoke:
function: databricks:getDataQualityMonitor
arguments:
objectType: schema
objectId: ${this.schemaInfo.schemaId}
Using getDataQualityMonitor
Two invocation forms are available. The direct form accepts plain arguments and either blocks until the result value is available, or returns a Promise-wrapped result. The output form accepts Input-wrapped arguments and returns an Output-wrapped result.
function getDataQualityMonitor(args: GetDataQualityMonitorArgs, opts?: InvokeOptions): Promise<GetDataQualityMonitorResult>
function getDataQualityMonitorOutput(args: GetDataQualityMonitorOutputArgs, opts?: InvokeOptions): Output<GetDataQualityMonitorResult>def get_data_quality_monitor(object_id: Optional[str] = None,
object_type: Optional[str] = None,
opts: Optional[InvokeOptions] = None) -> GetDataQualityMonitorResult
def get_data_quality_monitor_output(object_id: Optional[pulumi.Input[str]] = None,
object_type: Optional[pulumi.Input[str]] = None,
opts: Optional[InvokeOptions] = None) -> Output[GetDataQualityMonitorResult]func LookupDataQualityMonitor(ctx *Context, args *LookupDataQualityMonitorArgs, opts ...InvokeOption) (*LookupDataQualityMonitorResult, error)
func LookupDataQualityMonitorOutput(ctx *Context, args *LookupDataQualityMonitorOutputArgs, opts ...InvokeOption) LookupDataQualityMonitorResultOutput> Note: This function is named LookupDataQualityMonitor in the Go SDK.
public static class GetDataQualityMonitor
{
public static Task<GetDataQualityMonitorResult> InvokeAsync(GetDataQualityMonitorArgs args, InvokeOptions? opts = null)
public static Output<GetDataQualityMonitorResult> Invoke(GetDataQualityMonitorInvokeArgs args, InvokeOptions? opts = null)
}public static CompletableFuture<GetDataQualityMonitorResult> getDataQualityMonitor(GetDataQualityMonitorArgs args, InvokeOptions options)
public static Output<GetDataQualityMonitorResult> getDataQualityMonitor(GetDataQualityMonitorArgs args, InvokeOptions options)
fn::invoke:
function: databricks:index/getDataQualityMonitor:getDataQualityMonitor
arguments:
# arguments dictionaryThe following arguments are supported:
- Object
Id string The UUID of the request object. It is
schema_idforschema, andtable_idfortable.Find the
schema_idfrom either:- The [schema_id](https://docs.databricks.com/api/workspace/schemas/get#schema_id) of the
Schemasresource. - In Catalog Explorer > select the
schema> go to theDetailstab > theSchema IDfield.
Find the
table_idfrom either:- The [table_id](https://docs.databricks.com/api/workspace/tables/get#table_id) of the
Tablesresource. - In Catalog Explorer > select the
table> go to theDetailstab > theTable IDfield
- The [schema_id](https://docs.databricks.com/api/workspace/schemas/get#schema_id) of the
- Object
Type string - The type of the monitored object. Can be one of the following:
schemaortable
- Object
Id string The UUID of the request object. It is
schema_idforschema, andtable_idfortable.Find the
schema_idfrom either:- The [schema_id](https://docs.databricks.com/api/workspace/schemas/get#schema_id) of the
Schemasresource. - In Catalog Explorer > select the
schema> go to theDetailstab > theSchema IDfield.
Find the
table_idfrom either:- The [table_id](https://docs.databricks.com/api/workspace/tables/get#table_id) of the
Tablesresource. - In Catalog Explorer > select the
table> go to theDetailstab > theTable IDfield
- The [schema_id](https://docs.databricks.com/api/workspace/schemas/get#schema_id) of the
- Object
Type string - The type of the monitored object. Can be one of the following:
schemaortable
- object
Id String The UUID of the request object. It is
schema_idforschema, andtable_idfortable.Find the
schema_idfrom either:- The [schema_id](https://docs.databricks.com/api/workspace/schemas/get#schema_id) of the
Schemasresource. - In Catalog Explorer > select the
schema> go to theDetailstab > theSchema IDfield.
Find the
table_idfrom either:- The [table_id](https://docs.databricks.com/api/workspace/tables/get#table_id) of the
Tablesresource. - In Catalog Explorer > select the
table> go to theDetailstab > theTable IDfield
- The [schema_id](https://docs.databricks.com/api/workspace/schemas/get#schema_id) of the
- object
Type String - The type of the monitored object. Can be one of the following:
schemaortable
- object
Id string The UUID of the request object. It is
schema_idforschema, andtable_idfortable.Find the
schema_idfrom either:- The [schema_id](https://docs.databricks.com/api/workspace/schemas/get#schema_id) of the
Schemasresource. - In Catalog Explorer > select the
schema> go to theDetailstab > theSchema IDfield.
Find the
table_idfrom either:- The [table_id](https://docs.databricks.com/api/workspace/tables/get#table_id) of the
Tablesresource. - In Catalog Explorer > select the
table> go to theDetailstab > theTable IDfield
- The [schema_id](https://docs.databricks.com/api/workspace/schemas/get#schema_id) of the
- object
Type string - The type of the monitored object. Can be one of the following:
schemaortable
- object_
id str The UUID of the request object. It is
schema_idforschema, andtable_idfortable.Find the
schema_idfrom either:- The [schema_id](https://docs.databricks.com/api/workspace/schemas/get#schema_id) of the
Schemasresource. - In Catalog Explorer > select the
schema> go to theDetailstab > theSchema IDfield.
Find the
table_idfrom either:- The [table_id](https://docs.databricks.com/api/workspace/tables/get#table_id) of the
Tablesresource. - In Catalog Explorer > select the
table> go to theDetailstab > theTable IDfield
- The [schema_id](https://docs.databricks.com/api/workspace/schemas/get#schema_id) of the
- object_
type str - The type of the monitored object. Can be one of the following:
schemaortable
- object
Id String The UUID of the request object. It is
schema_idforschema, andtable_idfortable.Find the
schema_idfrom either:- The [schema_id](https://docs.databricks.com/api/workspace/schemas/get#schema_id) of the
Schemasresource. - In Catalog Explorer > select the
schema> go to theDetailstab > theSchema IDfield.
Find the
table_idfrom either:- The [table_id](https://docs.databricks.com/api/workspace/tables/get#table_id) of the
Tablesresource. - In Catalog Explorer > select the
table> go to theDetailstab > theTable IDfield
- The [schema_id](https://docs.databricks.com/api/workspace/schemas/get#schema_id) of the
- object
Type String - The type of the monitored object. Can be one of the following:
schemaortable
getDataQualityMonitor Result
The following output properties are available:
- Anomaly
Detection GetConfig Data Quality Monitor Anomaly Detection Config - (AnomalyDetectionConfig) - Anomaly Detection Configuration, applicable to
schemaobject types - Data
Profiling GetConfig Data Quality Monitor Data Profiling Config - (DataProfilingConfig) - Data Profiling Configuration, applicable to
tableobject types. Exactly oneAnalysis Configurationmust be present - Id string
- The provider-assigned unique ID for this managed resource.
- Object
Id string - (string) - The UUID of the request object. It is
schema_idforschema, andtable_idfortable. - Object
Type string - (string) - The type of the monitored object. Can be one of the following:
schemaortable
- Anomaly
Detection GetConfig Data Quality Monitor Anomaly Detection Config - (AnomalyDetectionConfig) - Anomaly Detection Configuration, applicable to
schemaobject types - Data
Profiling GetConfig Data Quality Monitor Data Profiling Config - (DataProfilingConfig) - Data Profiling Configuration, applicable to
tableobject types. Exactly oneAnalysis Configurationmust be present - Id string
- The provider-assigned unique ID for this managed resource.
- Object
Id string - (string) - The UUID of the request object. It is
schema_idforschema, andtable_idfortable. - Object
Type string - (string) - The type of the monitored object. Can be one of the following:
schemaortable
- anomaly
Detection GetConfig Data Quality Monitor Anomaly Detection Config - (AnomalyDetectionConfig) - Anomaly Detection Configuration, applicable to
schemaobject types - data
Profiling GetConfig Data Quality Monitor Data Profiling Config - (DataProfilingConfig) - Data Profiling Configuration, applicable to
tableobject types. Exactly oneAnalysis Configurationmust be present - id String
- The provider-assigned unique ID for this managed resource.
- object
Id String - (string) - The UUID of the request object. It is
schema_idforschema, andtable_idfortable. - object
Type String - (string) - The type of the monitored object. Can be one of the following:
schemaortable
- anomaly
Detection GetConfig Data Quality Monitor Anomaly Detection Config - (AnomalyDetectionConfig) - Anomaly Detection Configuration, applicable to
schemaobject types - data
Profiling GetConfig Data Quality Monitor Data Profiling Config - (DataProfilingConfig) - Data Profiling Configuration, applicable to
tableobject types. Exactly oneAnalysis Configurationmust be present - id string
- The provider-assigned unique ID for this managed resource.
- object
Id string - (string) - The UUID of the request object. It is
schema_idforschema, andtable_idfortable. - object
Type string - (string) - The type of the monitored object. Can be one of the following:
schemaortable
- anomaly_
detection_ Getconfig Data Quality Monitor Anomaly Detection Config - (AnomalyDetectionConfig) - Anomaly Detection Configuration, applicable to
schemaobject types - data_
profiling_ Getconfig Data Quality Monitor Data Profiling Config - (DataProfilingConfig) - Data Profiling Configuration, applicable to
tableobject types. Exactly oneAnalysis Configurationmust be present - id str
- The provider-assigned unique ID for this managed resource.
- object_
id str - (string) - The UUID of the request object. It is
schema_idforschema, andtable_idfortable. - object_
type str - (string) - The type of the monitored object. Can be one of the following:
schemaortable
- anomaly
Detection Property MapConfig - (AnomalyDetectionConfig) - Anomaly Detection Configuration, applicable to
schemaobject types - data
Profiling Property MapConfig - (DataProfilingConfig) - Data Profiling Configuration, applicable to
tableobject types. Exactly oneAnalysis Configurationmust be present - id String
- The provider-assigned unique ID for this managed resource.
- object
Id String - (string) - The UUID of the request object. It is
schema_idforschema, andtable_idfortable. - object
Type String - (string) - The type of the monitored object. Can be one of the following:
schemaortable
Supporting Types
GetDataQualityMonitorDataProfilingConfig
- Dashboard
Id string - (string) - Id of dashboard that visualizes the computed metrics. This can be empty if the monitor is in PENDING state
- Drift
Metrics stringTable Name - (string) - Table that stores drift metrics data. Format:
catalog.schema.table_name - Effective
Warehouse stringId - (string) - The warehouse for dashboard creation
- Latest
Monitor stringFailure Message - (string) - The latest error message for a monitor failure
- Monitor
Version int - (integer) - Represents the current monitor configuration version in use. The version will be represented in a numeric fashion (1,2,3...). The field has flexibility to take on negative values, which can indicate corrupted monitor_version numbers
- Monitored
Table stringName - (string) - Unity Catalog table to monitor. Format:
catalog.schema.table_name - Output
Schema stringId - (string) - ID of the schema where output tables are created
- Profile
Metrics stringTable Name - (string) - Table that stores profile metrics data. Format:
catalog.schema.table_name - Status string
- (string) - The data profiling monitor status. Possible values are:
DATA_PROFILING_STATUS_ACTIVE,DATA_PROFILING_STATUS_DELETE_PENDING,DATA_PROFILING_STATUS_ERROR,DATA_PROFILING_STATUS_FAILED,DATA_PROFILING_STATUS_PENDING - Assets
Dir string - (string) - Field for specifying the absolute path to a custom directory to store data-monitoring assets. Normally prepopulated to a default user location via UI and Python APIs
- Baseline
Table stringName - (string) - Baseline table name.
Baseline data is used to compute drift from the data in the monitored
table_name. The baseline table and the monitored table shall have the same schema - Custom
Metrics List<GetData Quality Monitor Data Profiling Config Custom Metric> - (list of DataProfilingCustomMetric) - Custom metrics
- Inference
Log GetData Quality Monitor Data Profiling Config Inference Log - (InferenceLogConfig) -
Analysis Configurationfor monitoring inference log tables - Notification
Settings GetData Quality Monitor Data Profiling Config Notification Settings - (NotificationSettings) - Field for specifying notification settings
- Schedule
Get
Data Quality Monitor Data Profiling Config Schedule - (CronSchedule) - The cron schedule
- Skip
Builtin boolDashboard - (boolean) - Whether to skip creating a default dashboard summarizing data quality metrics
- Slicing
Exprs List<string> - (list of string) - List of column expressions to slice data with for targeted analysis. The data is grouped by
each expression independently, resulting in a separate slice for each predicate and its
complements. For example
slicing_exprs=[“col_1”, “col_2 > 10”]will generate the following slices: two slices forcol_2 </span>> 10(True and False), and one slice per unique value incol1. For high-cardinality columns, only the top 100 unique values by frequency will generate slices - Snapshot
Get
Data Quality Monitor Data Profiling Config Snapshot - (SnapshotConfig) -
Analysis Configurationfor monitoring snapshot tables - Time
Series GetData Quality Monitor Data Profiling Config Time Series - (TimeSeriesConfig) -
Analysis Configurationfor monitoring time series tables - Warehouse
Id string - (string) - Optional argument to specify the warehouse for dashboard creation. If not specified, the first running warehouse will be used
- Dashboard
Id string - (string) - Id of dashboard that visualizes the computed metrics. This can be empty if the monitor is in PENDING state
- Drift
Metrics stringTable Name - (string) - Table that stores drift metrics data. Format:
catalog.schema.table_name - Effective
Warehouse stringId - (string) - The warehouse for dashboard creation
- Latest
Monitor stringFailure Message - (string) - The latest error message for a monitor failure
- Monitor
Version int - (integer) - Represents the current monitor configuration version in use. The version will be represented in a numeric fashion (1,2,3...). The field has flexibility to take on negative values, which can indicate corrupted monitor_version numbers
- Monitored
Table stringName - (string) - Unity Catalog table to monitor. Format:
catalog.schema.table_name - Output
Schema stringId - (string) - ID of the schema where output tables are created
- Profile
Metrics stringTable Name - (string) - Table that stores profile metrics data. Format:
catalog.schema.table_name - Status string
- (string) - The data profiling monitor status. Possible values are:
DATA_PROFILING_STATUS_ACTIVE,DATA_PROFILING_STATUS_DELETE_PENDING,DATA_PROFILING_STATUS_ERROR,DATA_PROFILING_STATUS_FAILED,DATA_PROFILING_STATUS_PENDING - Assets
Dir string - (string) - Field for specifying the absolute path to a custom directory to store data-monitoring assets. Normally prepopulated to a default user location via UI and Python APIs
- Baseline
Table stringName - (string) - Baseline table name.
Baseline data is used to compute drift from the data in the monitored
table_name. The baseline table and the monitored table shall have the same schema - Custom
Metrics []GetData Quality Monitor Data Profiling Config Custom Metric - (list of DataProfilingCustomMetric) - Custom metrics
- Inference
Log GetData Quality Monitor Data Profiling Config Inference Log - (InferenceLogConfig) -
Analysis Configurationfor monitoring inference log tables - Notification
Settings GetData Quality Monitor Data Profiling Config Notification Settings - (NotificationSettings) - Field for specifying notification settings
- Schedule
Get
Data Quality Monitor Data Profiling Config Schedule - (CronSchedule) - The cron schedule
- Skip
Builtin boolDashboard - (boolean) - Whether to skip creating a default dashboard summarizing data quality metrics
- Slicing
Exprs []string - (list of string) - List of column expressions to slice data with for targeted analysis. The data is grouped by
each expression independently, resulting in a separate slice for each predicate and its
complements. For example
slicing_exprs=[“col_1”, “col_2 > 10”]will generate the following slices: two slices forcol_2 </span>> 10(True and False), and one slice per unique value incol1. For high-cardinality columns, only the top 100 unique values by frequency will generate slices - Snapshot
Get
Data Quality Monitor Data Profiling Config Snapshot - (SnapshotConfig) -
Analysis Configurationfor monitoring snapshot tables - Time
Series GetData Quality Monitor Data Profiling Config Time Series - (TimeSeriesConfig) -
Analysis Configurationfor monitoring time series tables - Warehouse
Id string - (string) - Optional argument to specify the warehouse for dashboard creation. If not specified, the first running warehouse will be used
- dashboard
Id String - (string) - Id of dashboard that visualizes the computed metrics. This can be empty if the monitor is in PENDING state
- drift
Metrics StringTable Name - (string) - Table that stores drift metrics data. Format:
catalog.schema.table_name - effective
Warehouse StringId - (string) - The warehouse for dashboard creation
- latest
Monitor StringFailure Message - (string) - The latest error message for a monitor failure
- monitor
Version Integer - (integer) - Represents the current monitor configuration version in use. The version will be represented in a numeric fashion (1,2,3...). The field has flexibility to take on negative values, which can indicate corrupted monitor_version numbers
- monitored
Table StringName - (string) - Unity Catalog table to monitor. Format:
catalog.schema.table_name - output
Schema StringId - (string) - ID of the schema where output tables are created
- profile
Metrics StringTable Name - (string) - Table that stores profile metrics data. Format:
catalog.schema.table_name - status String
- (string) - The data profiling monitor status. Possible values are:
DATA_PROFILING_STATUS_ACTIVE,DATA_PROFILING_STATUS_DELETE_PENDING,DATA_PROFILING_STATUS_ERROR,DATA_PROFILING_STATUS_FAILED,DATA_PROFILING_STATUS_PENDING - assets
Dir String - (string) - Field for specifying the absolute path to a custom directory to store data-monitoring assets. Normally prepopulated to a default user location via UI and Python APIs
- baseline
Table StringName - (string) - Baseline table name.
Baseline data is used to compute drift from the data in the monitored
table_name. The baseline table and the monitored table shall have the same schema - custom
Metrics List<GetData Quality Monitor Data Profiling Config Custom Metric> - (list of DataProfilingCustomMetric) - Custom metrics
- inference
Log GetData Quality Monitor Data Profiling Config Inference Log - (InferenceLogConfig) -
Analysis Configurationfor monitoring inference log tables - notification
Settings GetData Quality Monitor Data Profiling Config Notification Settings - (NotificationSettings) - Field for specifying notification settings
- schedule
Get
Data Quality Monitor Data Profiling Config Schedule - (CronSchedule) - The cron schedule
- skip
Builtin BooleanDashboard - (boolean) - Whether to skip creating a default dashboard summarizing data quality metrics
- slicing
Exprs List<String> - (list of string) - List of column expressions to slice data with for targeted analysis. The data is grouped by
each expression independently, resulting in a separate slice for each predicate and its
complements. For example
slicing_exprs=[“col_1”, “col_2 > 10”]will generate the following slices: two slices forcol_2 </span>> 10(True and False), and one slice per unique value incol1. For high-cardinality columns, only the top 100 unique values by frequency will generate slices - snapshot
Get
Data Quality Monitor Data Profiling Config Snapshot - (SnapshotConfig) -
Analysis Configurationfor monitoring snapshot tables - time
Series GetData Quality Monitor Data Profiling Config Time Series - (TimeSeriesConfig) -
Analysis Configurationfor monitoring time series tables - warehouse
Id String - (string) - Optional argument to specify the warehouse for dashboard creation. If not specified, the first running warehouse will be used
- dashboard
Id string - (string) - Id of dashboard that visualizes the computed metrics. This can be empty if the monitor is in PENDING state
- drift
Metrics stringTable Name - (string) - Table that stores drift metrics data. Format:
catalog.schema.table_name - effective
Warehouse stringId - (string) - The warehouse for dashboard creation
- latest
Monitor stringFailure Message - (string) - The latest error message for a monitor failure
- monitor
Version number - (integer) - Represents the current monitor configuration version in use. The version will be represented in a numeric fashion (1,2,3...). The field has flexibility to take on negative values, which can indicate corrupted monitor_version numbers
- monitored
Table stringName - (string) - Unity Catalog table to monitor. Format:
catalog.schema.table_name - output
Schema stringId - (string) - ID of the schema where output tables are created
- profile
Metrics stringTable Name - (string) - Table that stores profile metrics data. Format:
catalog.schema.table_name - status string
- (string) - The data profiling monitor status. Possible values are:
DATA_PROFILING_STATUS_ACTIVE,DATA_PROFILING_STATUS_DELETE_PENDING,DATA_PROFILING_STATUS_ERROR,DATA_PROFILING_STATUS_FAILED,DATA_PROFILING_STATUS_PENDING - assets
Dir string - (string) - Field for specifying the absolute path to a custom directory to store data-monitoring assets. Normally prepopulated to a default user location via UI and Python APIs
- baseline
Table stringName - (string) - Baseline table name.
Baseline data is used to compute drift from the data in the monitored
table_name. The baseline table and the monitored table shall have the same schema - custom
Metrics GetData Quality Monitor Data Profiling Config Custom Metric[] - (list of DataProfilingCustomMetric) - Custom metrics
- inference
Log GetData Quality Monitor Data Profiling Config Inference Log - (InferenceLogConfig) -
Analysis Configurationfor monitoring inference log tables - notification
Settings GetData Quality Monitor Data Profiling Config Notification Settings - (NotificationSettings) - Field for specifying notification settings
- schedule
Get
Data Quality Monitor Data Profiling Config Schedule - (CronSchedule) - The cron schedule
- skip
Builtin booleanDashboard - (boolean) - Whether to skip creating a default dashboard summarizing data quality metrics
- slicing
Exprs string[] - (list of string) - List of column expressions to slice data with for targeted analysis. The data is grouped by
each expression independently, resulting in a separate slice for each predicate and its
complements. For example
slicing_exprs=[“col_1”, “col_2 > 10”]will generate the following slices: two slices forcol_2 </span>> 10(True and False), and one slice per unique value incol1. For high-cardinality columns, only the top 100 unique values by frequency will generate slices - snapshot
Get
Data Quality Monitor Data Profiling Config Snapshot - (SnapshotConfig) -
Analysis Configurationfor monitoring snapshot tables - time
Series GetData Quality Monitor Data Profiling Config Time Series - (TimeSeriesConfig) -
Analysis Configurationfor monitoring time series tables - warehouse
Id string - (string) - Optional argument to specify the warehouse for dashboard creation. If not specified, the first running warehouse will be used
- dashboard_
id str - (string) - Id of dashboard that visualizes the computed metrics. This can be empty if the monitor is in PENDING state
- drift_
metrics_ strtable_ name - (string) - Table that stores drift metrics data. Format:
catalog.schema.table_name - effective_
warehouse_ strid - (string) - The warehouse for dashboard creation
- latest_
monitor_ strfailure_ message - (string) - The latest error message for a monitor failure
- monitor_
version int - (integer) - Represents the current monitor configuration version in use. The version will be represented in a numeric fashion (1,2,3...). The field has flexibility to take on negative values, which can indicate corrupted monitor_version numbers
- monitored_
table_ strname - (string) - Unity Catalog table to monitor. Format:
catalog.schema.table_name - output_
schema_ strid - (string) - ID of the schema where output tables are created
- profile_
metrics_ strtable_ name - (string) - Table that stores profile metrics data. Format:
catalog.schema.table_name - status str
- (string) - The data profiling monitor status. Possible values are:
DATA_PROFILING_STATUS_ACTIVE,DATA_PROFILING_STATUS_DELETE_PENDING,DATA_PROFILING_STATUS_ERROR,DATA_PROFILING_STATUS_FAILED,DATA_PROFILING_STATUS_PENDING - assets_
dir str - (string) - Field for specifying the absolute path to a custom directory to store data-monitoring assets. Normally prepopulated to a default user location via UI and Python APIs
- baseline_
table_ strname - (string) - Baseline table name.
Baseline data is used to compute drift from the data in the monitored
table_name. The baseline table and the monitored table shall have the same schema - custom_
metrics Sequence[GetData Quality Monitor Data Profiling Config Custom Metric] - (list of DataProfilingCustomMetric) - Custom metrics
- inference_
log GetData Quality Monitor Data Profiling Config Inference Log - (InferenceLogConfig) -
Analysis Configurationfor monitoring inference log tables - notification_
settings GetData Quality Monitor Data Profiling Config Notification Settings - (NotificationSettings) - Field for specifying notification settings
- schedule
Get
Data Quality Monitor Data Profiling Config Schedule - (CronSchedule) - The cron schedule
- skip_
builtin_ booldashboard - (boolean) - Whether to skip creating a default dashboard summarizing data quality metrics
- slicing_
exprs Sequence[str] - (list of string) - List of column expressions to slice data with for targeted analysis. The data is grouped by
each expression independently, resulting in a separate slice for each predicate and its
complements. For example
slicing_exprs=[“col_1”, “col_2 > 10”]will generate the following slices: two slices forcol_2 </span>> 10(True and False), and one slice per unique value incol1. For high-cardinality columns, only the top 100 unique values by frequency will generate slices - snapshot
Get
Data Quality Monitor Data Profiling Config Snapshot - (SnapshotConfig) -
Analysis Configurationfor monitoring snapshot tables - time_
series GetData Quality Monitor Data Profiling Config Time Series - (TimeSeriesConfig) -
Analysis Configurationfor monitoring time series tables - warehouse_
id str - (string) - Optional argument to specify the warehouse for dashboard creation. If not specified, the first running warehouse will be used
- dashboard
Id String - (string) - Id of dashboard that visualizes the computed metrics. This can be empty if the monitor is in PENDING state
- drift
Metrics StringTable Name - (string) - Table that stores drift metrics data. Format:
catalog.schema.table_name - effective
Warehouse StringId - (string) - The warehouse for dashboard creation
- latest
Monitor StringFailure Message - (string) - The latest error message for a monitor failure
- monitor
Version Number - (integer) - Represents the current monitor configuration version in use. The version will be represented in a numeric fashion (1,2,3...). The field has flexibility to take on negative values, which can indicate corrupted monitor_version numbers
- monitored
Table StringName - (string) - Unity Catalog table to monitor. Format:
catalog.schema.table_name - output
Schema StringId - (string) - ID of the schema where output tables are created
- profile
Metrics StringTable Name - (string) - Table that stores profile metrics data. Format:
catalog.schema.table_name - status String
- (string) - The data profiling monitor status. Possible values are:
DATA_PROFILING_STATUS_ACTIVE,DATA_PROFILING_STATUS_DELETE_PENDING,DATA_PROFILING_STATUS_ERROR,DATA_PROFILING_STATUS_FAILED,DATA_PROFILING_STATUS_PENDING - assets
Dir String - (string) - Field for specifying the absolute path to a custom directory to store data-monitoring assets. Normally prepopulated to a default user location via UI and Python APIs
- baseline
Table StringName - (string) - Baseline table name.
Baseline data is used to compute drift from the data in the monitored
table_name. The baseline table and the monitored table shall have the same schema - custom
Metrics List<Property Map> - (list of DataProfilingCustomMetric) - Custom metrics
- inference
Log Property Map - (InferenceLogConfig) -
Analysis Configurationfor monitoring inference log tables - notification
Settings Property Map - (NotificationSettings) - Field for specifying notification settings
- schedule Property Map
- (CronSchedule) - The cron schedule
- skip
Builtin BooleanDashboard - (boolean) - Whether to skip creating a default dashboard summarizing data quality metrics
- slicing
Exprs List<String> - (list of string) - List of column expressions to slice data with for targeted analysis. The data is grouped by
each expression independently, resulting in a separate slice for each predicate and its
complements. For example
slicing_exprs=[“col_1”, “col_2 > 10”]will generate the following slices: two slices forcol_2 </span>> 10(True and False), and one slice per unique value incol1. For high-cardinality columns, only the top 100 unique values by frequency will generate slices - snapshot Property Map
- (SnapshotConfig) -
Analysis Configurationfor monitoring snapshot tables - time
Series Property Map - (TimeSeriesConfig) -
Analysis Configurationfor monitoring time series tables - warehouse
Id String - (string) - Optional argument to specify the warehouse for dashboard creation. If not specified, the first running warehouse will be used
GetDataQualityMonitorDataProfilingConfigCustomMetric
- Definition string
- (string) - Jinja template for a SQL expression that specifies how to compute the metric. See create metric definition
- Input
Columns List<string> - (list of string) - A list of column names in the input table the metric should be computed for.
Can use
":table"to indicate that the metric needs information from multiple columns - Name string
- (string) - Name of the metric in the output tables
- Output
Data stringType - (string) - The output type of the custom metric
- Type string
- (string) - The type of the custom metric. Possible values are:
DATA_PROFILING_CUSTOM_METRIC_TYPE_AGGREGATE,DATA_PROFILING_CUSTOM_METRIC_TYPE_DERIVED,DATA_PROFILING_CUSTOM_METRIC_TYPE_DRIFT
- Definition string
- (string) - Jinja template for a SQL expression that specifies how to compute the metric. See create metric definition
- Input
Columns []string - (list of string) - A list of column names in the input table the metric should be computed for.
Can use
":table"to indicate that the metric needs information from multiple columns - Name string
- (string) - Name of the metric in the output tables
- Output
Data stringType - (string) - The output type of the custom metric
- Type string
- (string) - The type of the custom metric. Possible values are:
DATA_PROFILING_CUSTOM_METRIC_TYPE_AGGREGATE,DATA_PROFILING_CUSTOM_METRIC_TYPE_DERIVED,DATA_PROFILING_CUSTOM_METRIC_TYPE_DRIFT
- definition String
- (string) - Jinja template for a SQL expression that specifies how to compute the metric. See create metric definition
- input
Columns List<String> - (list of string) - A list of column names in the input table the metric should be computed for.
Can use
":table"to indicate that the metric needs information from multiple columns - name String
- (string) - Name of the metric in the output tables
- output
Data StringType - (string) - The output type of the custom metric
- type String
- (string) - The type of the custom metric. Possible values are:
DATA_PROFILING_CUSTOM_METRIC_TYPE_AGGREGATE,DATA_PROFILING_CUSTOM_METRIC_TYPE_DERIVED,DATA_PROFILING_CUSTOM_METRIC_TYPE_DRIFT
- definition string
- (string) - Jinja template for a SQL expression that specifies how to compute the metric. See create metric definition
- input
Columns string[] - (list of string) - A list of column names in the input table the metric should be computed for.
Can use
":table"to indicate that the metric needs information from multiple columns - name string
- (string) - Name of the metric in the output tables
- output
Data stringType - (string) - The output type of the custom metric
- type string
- (string) - The type of the custom metric. Possible values are:
DATA_PROFILING_CUSTOM_METRIC_TYPE_AGGREGATE,DATA_PROFILING_CUSTOM_METRIC_TYPE_DERIVED,DATA_PROFILING_CUSTOM_METRIC_TYPE_DRIFT
- definition str
- (string) - Jinja template for a SQL expression that specifies how to compute the metric. See create metric definition
- input_
columns Sequence[str] - (list of string) - A list of column names in the input table the metric should be computed for.
Can use
":table"to indicate that the metric needs information from multiple columns - name str
- (string) - Name of the metric in the output tables
- output_
data_ strtype - (string) - The output type of the custom metric
- type str
- (string) - The type of the custom metric. Possible values are:
DATA_PROFILING_CUSTOM_METRIC_TYPE_AGGREGATE,DATA_PROFILING_CUSTOM_METRIC_TYPE_DERIVED,DATA_PROFILING_CUSTOM_METRIC_TYPE_DRIFT
- definition String
- (string) - Jinja template for a SQL expression that specifies how to compute the metric. See create metric definition
- input
Columns List<String> - (list of string) - A list of column names in the input table the metric should be computed for.
Can use
":table"to indicate that the metric needs information from multiple columns - name String
- (string) - Name of the metric in the output tables
- output
Data StringType - (string) - The output type of the custom metric
- type String
- (string) - The type of the custom metric. Possible values are:
DATA_PROFILING_CUSTOM_METRIC_TYPE_AGGREGATE,DATA_PROFILING_CUSTOM_METRIC_TYPE_DERIVED,DATA_PROFILING_CUSTOM_METRIC_TYPE_DRIFT
GetDataQualityMonitorDataProfilingConfigInferenceLog
- Granularities List<string>
- (list of string) - List of granularities to use when aggregating data into time windows based on their timestamp
- Model
Id stringColumn - (string) - Column for the model identifier
- Prediction
Column string - (string) - Column for the prediction
- Problem
Type string - (string) - Problem type the model aims to solve. Possible values are:
INFERENCE_PROBLEM_TYPE_CLASSIFICATION,INFERENCE_PROBLEM_TYPE_REGRESSION - Timestamp
Column string - (string) - Column for the timestamp
- Label
Column string - (string) - Column for the label
- Granularities []string
- (list of string) - List of granularities to use when aggregating data into time windows based on their timestamp
- Model
Id stringColumn - (string) - Column for the model identifier
- Prediction
Column string - (string) - Column for the prediction
- Problem
Type string - (string) - Problem type the model aims to solve. Possible values are:
INFERENCE_PROBLEM_TYPE_CLASSIFICATION,INFERENCE_PROBLEM_TYPE_REGRESSION - Timestamp
Column string - (string) - Column for the timestamp
- Label
Column string - (string) - Column for the label
- granularities List<String>
- (list of string) - List of granularities to use when aggregating data into time windows based on their timestamp
- model
Id StringColumn - (string) - Column for the model identifier
- prediction
Column String - (string) - Column for the prediction
- problem
Type String - (string) - Problem type the model aims to solve. Possible values are:
INFERENCE_PROBLEM_TYPE_CLASSIFICATION,INFERENCE_PROBLEM_TYPE_REGRESSION - timestamp
Column String - (string) - Column for the timestamp
- label
Column String - (string) - Column for the label
- granularities string[]
- (list of string) - List of granularities to use when aggregating data into time windows based on their timestamp
- model
Id stringColumn - (string) - Column for the model identifier
- prediction
Column string - (string) - Column for the prediction
- problem
Type string - (string) - Problem type the model aims to solve. Possible values are:
INFERENCE_PROBLEM_TYPE_CLASSIFICATION,INFERENCE_PROBLEM_TYPE_REGRESSION - timestamp
Column string - (string) - Column for the timestamp
- label
Column string - (string) - Column for the label
- granularities Sequence[str]
- (list of string) - List of granularities to use when aggregating data into time windows based on their timestamp
- model_
id_ strcolumn - (string) - Column for the model identifier
- prediction_
column str - (string) - Column for the prediction
- problem_
type str - (string) - Problem type the model aims to solve. Possible values are:
INFERENCE_PROBLEM_TYPE_CLASSIFICATION,INFERENCE_PROBLEM_TYPE_REGRESSION - timestamp_
column str - (string) - Column for the timestamp
- label_
column str - (string) - Column for the label
- granularities List<String>
- (list of string) - List of granularities to use when aggregating data into time windows based on their timestamp
- model
Id StringColumn - (string) - Column for the model identifier
- prediction
Column String - (string) - Column for the prediction
- problem
Type String - (string) - Problem type the model aims to solve. Possible values are:
INFERENCE_PROBLEM_TYPE_CLASSIFICATION,INFERENCE_PROBLEM_TYPE_REGRESSION - timestamp
Column String - (string) - Column for the timestamp
- label
Column String - (string) - Column for the label
GetDataQualityMonitorDataProfilingConfigNotificationSettings
- On
Failure GetData Quality Monitor Data Profiling Config Notification Settings On Failure - (NotificationDestination) - Destinations to send notifications on failure/timeout
- On
Failure GetData Quality Monitor Data Profiling Config Notification Settings On Failure - (NotificationDestination) - Destinations to send notifications on failure/timeout
- on
Failure GetData Quality Monitor Data Profiling Config Notification Settings On Failure - (NotificationDestination) - Destinations to send notifications on failure/timeout
- on
Failure GetData Quality Monitor Data Profiling Config Notification Settings On Failure - (NotificationDestination) - Destinations to send notifications on failure/timeout
- on_
failure GetData Quality Monitor Data Profiling Config Notification Settings On Failure - (NotificationDestination) - Destinations to send notifications on failure/timeout
- on
Failure Property Map - (NotificationDestination) - Destinations to send notifications on failure/timeout
GetDataQualityMonitorDataProfilingConfigNotificationSettingsOnFailure
- Email
Addresses List<string> - (list of string) - The list of email addresses to send the notification to. A maximum of 5 email addresses is supported
- Email
Addresses []string - (list of string) - The list of email addresses to send the notification to. A maximum of 5 email addresses is supported
- email
Addresses List<String> - (list of string) - The list of email addresses to send the notification to. A maximum of 5 email addresses is supported
- email
Addresses string[] - (list of string) - The list of email addresses to send the notification to. A maximum of 5 email addresses is supported
- email_
addresses Sequence[str] - (list of string) - The list of email addresses to send the notification to. A maximum of 5 email addresses is supported
- email
Addresses List<String> - (list of string) - The list of email addresses to send the notification to. A maximum of 5 email addresses is supported
GetDataQualityMonitorDataProfilingConfigSchedule
- Pause
Status string - (string) - Read only field that indicates whether the schedule is paused or not. Possible values are:
CRON_SCHEDULE_PAUSE_STATUS_PAUSED,CRON_SCHEDULE_PAUSE_STATUS_UNPAUSED - Quartz
Cron stringExpression - (string) - The expression that determines when to run the monitor. See examples
- Timezone
Id string - (string) - A Java timezone id. The schedule for a job will be resolved with respect to this timezone.
See
Java TimeZone <http://docs.oracle.com/javase/7/docs/api/java/util/TimeZone.html>_ for details. The timezone id (e.g.,America/Los_Angeles) in which to evaluate the quartz expression
- Pause
Status string - (string) - Read only field that indicates whether the schedule is paused or not. Possible values are:
CRON_SCHEDULE_PAUSE_STATUS_PAUSED,CRON_SCHEDULE_PAUSE_STATUS_UNPAUSED - Quartz
Cron stringExpression - (string) - The expression that determines when to run the monitor. See examples
- Timezone
Id string - (string) - A Java timezone id. The schedule for a job will be resolved with respect to this timezone.
See
Java TimeZone <http://docs.oracle.com/javase/7/docs/api/java/util/TimeZone.html>_ for details. The timezone id (e.g.,America/Los_Angeles) in which to evaluate the quartz expression
- pause
Status String - (string) - Read only field that indicates whether the schedule is paused or not. Possible values are:
CRON_SCHEDULE_PAUSE_STATUS_PAUSED,CRON_SCHEDULE_PAUSE_STATUS_UNPAUSED - quartz
Cron StringExpression - (string) - The expression that determines when to run the monitor. See examples
- timezone
Id String - (string) - A Java timezone id. The schedule for a job will be resolved with respect to this timezone.
See
Java TimeZone <http://docs.oracle.com/javase/7/docs/api/java/util/TimeZone.html>_ for details. The timezone id (e.g.,America/Los_Angeles) in which to evaluate the quartz expression
- pause
Status string - (string) - Read only field that indicates whether the schedule is paused or not. Possible values are:
CRON_SCHEDULE_PAUSE_STATUS_PAUSED,CRON_SCHEDULE_PAUSE_STATUS_UNPAUSED - quartz
Cron stringExpression - (string) - The expression that determines when to run the monitor. See examples
- timezone
Id string - (string) - A Java timezone id. The schedule for a job will be resolved with respect to this timezone.
See
Java TimeZone <http://docs.oracle.com/javase/7/docs/api/java/util/TimeZone.html>_ for details. The timezone id (e.g.,America/Los_Angeles) in which to evaluate the quartz expression
- pause_
status str - (string) - Read only field that indicates whether the schedule is paused or not. Possible values are:
CRON_SCHEDULE_PAUSE_STATUS_PAUSED,CRON_SCHEDULE_PAUSE_STATUS_UNPAUSED - quartz_
cron_ strexpression - (string) - The expression that determines when to run the monitor. See examples
- timezone_
id str - (string) - A Java timezone id. The schedule for a job will be resolved with respect to this timezone.
See
Java TimeZone <http://docs.oracle.com/javase/7/docs/api/java/util/TimeZone.html>_ for details. The timezone id (e.g.,America/Los_Angeles) in which to evaluate the quartz expression
- pause
Status String - (string) - Read only field that indicates whether the schedule is paused or not. Possible values are:
CRON_SCHEDULE_PAUSE_STATUS_PAUSED,CRON_SCHEDULE_PAUSE_STATUS_UNPAUSED - quartz
Cron StringExpression - (string) - The expression that determines when to run the monitor. See examples
- timezone
Id String - (string) - A Java timezone id. The schedule for a job will be resolved with respect to this timezone.
See
Java TimeZone <http://docs.oracle.com/javase/7/docs/api/java/util/TimeZone.html>_ for details. The timezone id (e.g.,America/Los_Angeles) in which to evaluate the quartz expression
GetDataQualityMonitorDataProfilingConfigTimeSeries
- Granularities List<string>
- (list of string) - List of granularities to use when aggregating data into time windows based on their timestamp
- Timestamp
Column string - (string) - Column for the timestamp
- Granularities []string
- (list of string) - List of granularities to use when aggregating data into time windows based on their timestamp
- Timestamp
Column string - (string) - Column for the timestamp
- granularities List<String>
- (list of string) - List of granularities to use when aggregating data into time windows based on their timestamp
- timestamp
Column String - (string) - Column for the timestamp
- granularities string[]
- (list of string) - List of granularities to use when aggregating data into time windows based on their timestamp
- timestamp
Column string - (string) - Column for the timestamp
- granularities Sequence[str]
- (list of string) - List of granularities to use when aggregating data into time windows based on their timestamp
- timestamp_
column str - (string) - Column for the timestamp
- granularities List<String>
- (list of string) - List of granularities to use when aggregating data into time windows based on their timestamp
- timestamp
Column String - (string) - Column for the timestamp
Package Details
- Repository
- databricks pulumi/pulumi-databricks
- License
- Apache-2.0
- Notes
- This Pulumi package is based on the
databricksTerraform Provider.
