// Code generated by private/model/cli/gen-api/main.go. DO NOT EDIT. package firehose import ( "fmt" "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/internal/awsutil" ) var _ aws.Config var _ = awsutil.Prettify // Describes hints for the buffering to perform before delivering data to the // destination. These options are treated as hints, and therefore Kinesis Data // Firehose might choose to use different values when it is optimal. The SizeInMBs // and IntervalInSeconds parameters are optional. However, if specify a value // for one of them, you must also provide a value for the other. type BufferingHints struct { _ struct{} `type:"structure"` // Buffer incoming data for the specified period of time, in seconds, before // delivering it to the destination. The default value is 300. This parameter // is optional but if you specify a value for it, you must also specify a value // for SizeInMBs, and vice versa. IntervalInSeconds *int64 `min:"60" type:"integer"` // Buffer incoming data to the specified size, in MiBs, before delivering it // to the destination. The default value is 5. This parameter is optional but // if you specify a value for it, you must also specify a value for IntervalInSeconds, // and vice versa. // // We recommend setting this parameter to a value greater than the amount of // data you typically ingest into the delivery stream in 10 seconds. For example, // if you typically ingest data at 1 MiB/sec, the value should be 10 MiB or // higher. SizeInMBs *int64 `min:"1" type:"integer"` } // String returns the string representation func (s BufferingHints) String() string { return awsutil.Prettify(s) } // Validate inspects the fields of the type to determine if they are valid. func (s *BufferingHints) Validate() error { invalidParams := aws.ErrInvalidParams{Context: "BufferingHints"} if s.IntervalInSeconds != nil && *s.IntervalInSeconds < 60 { invalidParams.Add(aws.NewErrParamMinValue("IntervalInSeconds", 60)) } if s.SizeInMBs != nil && *s.SizeInMBs < 1 { invalidParams.Add(aws.NewErrParamMinValue("SizeInMBs", 1)) } if invalidParams.Len() > 0 { return invalidParams } return nil } // Describes the Amazon CloudWatch logging options for your delivery stream. type CloudWatchLoggingOptions struct { _ struct{} `type:"structure"` // Enables or disables CloudWatch logging. Enabled *bool `type:"boolean"` // The CloudWatch group name for logging. This value is required if CloudWatch // logging is enabled. LogGroupName *string `type:"string"` // The CloudWatch log stream name for logging. This value is required if CloudWatch // logging is enabled. LogStreamName *string `type:"string"` } // String returns the string representation func (s CloudWatchLoggingOptions) String() string { return awsutil.Prettify(s) } // Describes a COPY command for Amazon Redshift. type CopyCommand struct { _ struct{} `type:"structure"` // Optional parameters to use with the Amazon Redshift COPY command. For more // information, see the "Optional Parameters" section of Amazon Redshift COPY // command (https://docs.aws.amazon.com/redshift/latest/dg/r_COPY.html). Some // possible examples that would apply to Kinesis Data Firehose are as follows: // // delimiter '\t' lzop; - fields are delimited with "\t" (TAB character) and // compressed using lzop. // // delimiter '|' - fields are delimited with "|" (this is the default delimiter). // // delimiter '|' escape - the delimiter should be escaped. // // fixedwidth 'venueid:3,venuename:25,venuecity:12,venuestate:2,venueseats:6' // - fields are fixed width in the source, with each width specified after every // column in the table. // // JSON 's3://mybucket/jsonpaths.txt' - data is in JSON format, and the path // specified is the format of the data. // // For more examples, see Amazon Redshift COPY command examples (https://docs.aws.amazon.com/redshift/latest/dg/r_COPY_command_examples.html). CopyOptions *string `type:"string"` // A comma-separated list of column names. DataTableColumns *string `type:"string"` // The name of the target table. The table must already exist in the database. // // DataTableName is a required field DataTableName *string `min:"1" type:"string" required:"true"` } // String returns the string representation func (s CopyCommand) String() string { return awsutil.Prettify(s) } // Validate inspects the fields of the type to determine if they are valid. func (s *CopyCommand) Validate() error { invalidParams := aws.ErrInvalidParams{Context: "CopyCommand"} if s.DataTableName == nil { invalidParams.Add(aws.NewErrParamRequired("DataTableName")) } if s.DataTableName != nil && len(*s.DataTableName) < 1 { invalidParams.Add(aws.NewErrParamMinLen("DataTableName", 1)) } if invalidParams.Len() > 0 { return invalidParams } return nil } // Specifies that you want Kinesis Data Firehose to convert data from the JSON // format to the Parquet or ORC format before writing it to Amazon S3. Kinesis // Data Firehose uses the serializer and deserializer that you specify, in addition // to the column information from the AWS Glue table, to deserialize your input // data from JSON and then serialize it to the Parquet or ORC format. For more // information, see Kinesis Data Firehose Record Format Conversion (https://docs.aws.amazon.com/firehose/latest/dev/record-format-conversion.html). type DataFormatConversionConfiguration struct { _ struct{} `type:"structure"` // Defaults to true. Set it to false if you want to disable format conversion // while preserving the configuration details. Enabled *bool `type:"boolean"` // Specifies the deserializer that you want Kinesis Data Firehose to use to // convert the format of your data from JSON. This parameter is required if // Enabled is set to true. InputFormatConfiguration *InputFormatConfiguration `type:"structure"` // Specifies the serializer that you want Kinesis Data Firehose to use to convert // the format of your data to the Parquet or ORC format. This parameter is required // if Enabled is set to true. OutputFormatConfiguration *OutputFormatConfiguration `type:"structure"` // Specifies the AWS Glue Data Catalog table that contains the column information. // This parameter is required if Enabled is set to true. SchemaConfiguration *SchemaConfiguration `type:"structure"` } // String returns the string representation func (s DataFormatConversionConfiguration) String() string { return awsutil.Prettify(s) } // Validate inspects the fields of the type to determine if they are valid. func (s *DataFormatConversionConfiguration) Validate() error { invalidParams := aws.ErrInvalidParams{Context: "DataFormatConversionConfiguration"} if s.OutputFormatConfiguration != nil { if err := s.OutputFormatConfiguration.Validate(); err != nil { invalidParams.AddNested("OutputFormatConfiguration", err.(aws.ErrInvalidParams)) } } if s.SchemaConfiguration != nil { if err := s.SchemaConfiguration.Validate(); err != nil { invalidParams.AddNested("SchemaConfiguration", err.(aws.ErrInvalidParams)) } } if invalidParams.Len() > 0 { return invalidParams } return nil } // Contains information about a delivery stream. type DeliveryStreamDescription struct { _ struct{} `type:"structure"` // The date and time that the delivery stream was created. CreateTimestamp *time.Time `type:"timestamp"` // The Amazon Resource Name (ARN) of the delivery stream. For more information, // see Amazon Resource Names (ARNs) and AWS Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html). // // DeliveryStreamARN is a required field DeliveryStreamARN *string `min:"1" type:"string" required:"true"` // Indicates the server-side encryption (SSE) status for the delivery stream. DeliveryStreamEncryptionConfiguration *DeliveryStreamEncryptionConfiguration `type:"structure"` // The name of the delivery stream. // // DeliveryStreamName is a required field DeliveryStreamName *string `min:"1" type:"string" required:"true"` // The status of the delivery stream. If the status of a delivery stream is // CREATING_FAILED, this status doesn't change, and you can't invoke CreateDeliveryStream // again on it. However, you can invoke the DeleteDeliveryStream operation to // delete it. // // DeliveryStreamStatus is a required field DeliveryStreamStatus DeliveryStreamStatus `type:"string" required:"true" enum:"true"` // The delivery stream type. This can be one of the following values: // // * DirectPut: Provider applications access the delivery stream directly. // // * KinesisStreamAsSource: The delivery stream uses a Kinesis data stream // as a source. // // DeliveryStreamType is a required field DeliveryStreamType DeliveryStreamType `type:"string" required:"true" enum:"true"` // The destinations. // // Destinations is a required field Destinations []DestinationDescription `type:"list" required:"true"` // Provides details in case one of the following operations fails due to an // error related to KMS: CreateDeliveryStream, DeleteDeliveryStream, StartDeliveryStreamEncryption, // StopDeliveryStreamEncryption. FailureDescription *FailureDescription `type:"structure"` // Indicates whether there are more destinations available to list. // // HasMoreDestinations is a required field HasMoreDestinations *bool `type:"boolean" required:"true"` // The date and time that the delivery stream was last updated. LastUpdateTimestamp *time.Time `type:"timestamp"` // If the DeliveryStreamType parameter is KinesisStreamAsSource, a SourceDescription // object describing the source Kinesis data stream. Source *SourceDescription `type:"structure"` // Each time the destination is updated for a delivery stream, the version ID // is changed, and the current version ID is required when updating the destination. // This is so that the service knows it is applying the changes to the correct // version of the delivery stream. // // VersionId is a required field VersionId *string `min:"1" type:"string" required:"true"` } // String returns the string representation func (s DeliveryStreamDescription) String() string { return awsutil.Prettify(s) } // Contains information about the server-side encryption (SSE) status for the // delivery stream, the type customer master key (CMK) in use, if any, and the // ARN of the CMK. You can get DeliveryStreamEncryptionConfiguration by invoking // the DescribeDeliveryStream operation. type DeliveryStreamEncryptionConfiguration struct { _ struct{} `type:"structure"` // Provides details in case one of the following operations fails due to an // error related to KMS: CreateDeliveryStream, DeleteDeliveryStream, StartDeliveryStreamEncryption, // StopDeliveryStreamEncryption. FailureDescription *FailureDescription `type:"structure"` // If KeyType is CUSTOMER_MANAGED_CMK, this field contains the ARN of the customer // managed CMK. If KeyType is AWS_OWNED_CMK, DeliveryStreamEncryptionConfiguration // doesn't contain a value for KeyARN. KeyARN *string `min:"1" type:"string"` // Indicates the type of customer master key (CMK) that is used for encryption. // The default setting is AWS_OWNED_CMK. For more information about CMKs, see // Customer Master Keys (CMKs) (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#master_keys). KeyType KeyType `type:"string" enum:"true"` // This is the server-side encryption (SSE) status for the delivery stream. // For a full description of the different values of this status, see StartDeliveryStreamEncryption // and StopDeliveryStreamEncryption. If this status is ENABLING_FAILED or DISABLING_FAILED, // it is the status of the most recent attempt to enable or disable SSE, respectively. Status DeliveryStreamEncryptionStatus `type:"string" enum:"true"` } // String returns the string representation func (s DeliveryStreamEncryptionConfiguration) String() string { return awsutil.Prettify(s) } // Specifies the type and Amazon Resource Name (ARN) of the CMK to use for Server-Side // Encryption (SSE). type DeliveryStreamEncryptionConfigurationInput struct { _ struct{} `type:"structure"` // If you set KeyType to CUSTOMER_MANAGED_CMK, you must specify the Amazon Resource // Name (ARN) of the CMK. If you set KeyType to AWS_OWNED_CMK, Kinesis Data // Firehose uses a service-account CMK. KeyARN *string `min:"1" type:"string"` // Indicates the type of customer master key (CMK) to use for encryption. The // default setting is AWS_OWNED_CMK. For more information about CMKs, see Customer // Master Keys (CMKs) (https://docs.aws.amazon.com/kms/latest/developerguide/concepts.html#master_keys). // When you invoke CreateDeliveryStream or StartDeliveryStreamEncryption with // KeyType set to CUSTOMER_MANAGED_CMK, Kinesis Data Firehose invokes the Amazon // KMS operation CreateGrant (https://docs.aws.amazon.com/kms/latest/APIReference/API_CreateGrant.html) // to create a grant that allows the Kinesis Data Firehose service to use the // customer managed CMK to perform encryption and decryption. Kinesis Data Firehose // manages that grant. // // When you invoke StartDeliveryStreamEncryption to change the CMK for a delivery // stream that is encrypted with a customer managed CMK, Kinesis Data Firehose // schedules the grant it had on the old CMK for retirement. // // You can use a CMK of type CUSTOMER_MANAGED_CMK to encrypt up to 500 delivery // streams. If a CreateDeliveryStream or StartDeliveryStreamEncryption operation // exceeds this limit, Kinesis Data Firehose throws a LimitExceededException. // // To encrypt your delivery stream, use symmetric CMKs. Kinesis Data Firehose // doesn't support asymmetric CMKs. For information about symmetric and asymmetric // CMKs, see About Symmetric and Asymmetric CMKs (https://docs.aws.amazon.com/kms/latest/developerguide/symm-asymm-concepts.html) // in the AWS Key Management Service developer guide. // // KeyType is a required field KeyType KeyType `type:"string" required:"true" enum:"true"` } // String returns the string representation func (s DeliveryStreamEncryptionConfigurationInput) String() string { return awsutil.Prettify(s) } // Validate inspects the fields of the type to determine if they are valid. func (s *DeliveryStreamEncryptionConfigurationInput) Validate() error { invalidParams := aws.ErrInvalidParams{Context: "DeliveryStreamEncryptionConfigurationInput"} if s.KeyARN != nil && len(*s.KeyARN) < 1 { invalidParams.Add(aws.NewErrParamMinLen("KeyARN", 1)) } if len(s.KeyType) == 0 { invalidParams.Add(aws.NewErrParamRequired("KeyType")) } if invalidParams.Len() > 0 { return invalidParams } return nil } // The deserializer you want Kinesis Data Firehose to use for converting the // input data from JSON. Kinesis Data Firehose then serializes the data to its // final format using the Serializer. Kinesis Data Firehose supports two types // of deserializers: the Apache Hive JSON SerDe (https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL#LanguageManualDDL-JSON) // and the OpenX JSON SerDe (https://github.com/rcongiu/Hive-JSON-Serde). type Deserializer struct { _ struct{} `type:"structure"` // The native Hive / HCatalog JsonSerDe. Used by Kinesis Data Firehose for deserializing // data, which means converting it from the JSON format in preparation for serializing // it to the Parquet or ORC format. This is one of two deserializers you can // choose, depending on which one offers the functionality you need. The other // option is the OpenX SerDe. HiveJsonSerDe *HiveJsonSerDe `type:"structure"` // The OpenX SerDe. Used by Kinesis Data Firehose for deserializing data, which // means converting it from the JSON format in preparation for serializing it // to the Parquet or ORC format. This is one of two deserializers you can choose, // depending on which one offers the functionality you need. The other option // is the native Hive / HCatalog JsonSerDe. OpenXJsonSerDe *OpenXJsonSerDe `type:"structure"` } // String returns the string representation func (s Deserializer) String() string { return awsutil.Prettify(s) } // Describes the destination for a delivery stream. type DestinationDescription struct { _ struct{} `type:"structure"` // The ID of the destination. // // DestinationId is a required field DestinationId *string `min:"1" type:"string" required:"true"` // The destination in Amazon ES. ElasticsearchDestinationDescription *ElasticsearchDestinationDescription `type:"structure"` // The destination in Amazon S3. ExtendedS3DestinationDescription *ExtendedS3DestinationDescription `type:"structure"` // The destination in Amazon Redshift. RedshiftDestinationDescription *RedshiftDestinationDescription `type:"structure"` // [Deprecated] The destination in Amazon S3. S3DestinationDescription *S3DestinationDescription `type:"structure"` // The destination in Splunk. SplunkDestinationDescription *SplunkDestinationDescription `type:"structure"` } // String returns the string representation func (s DestinationDescription) String() string { return awsutil.Prettify(s) } // Describes the buffering to perform before delivering data to the Amazon ES // destination. type ElasticsearchBufferingHints struct { _ struct{} `type:"structure"` // Buffer incoming data for the specified period of time, in seconds, before // delivering it to the destination. The default value is 300 (5 minutes). IntervalInSeconds *int64 `min:"60" type:"integer"` // Buffer incoming data to the specified size, in MBs, before delivering it // to the destination. The default value is 5. // // We recommend setting this parameter to a value greater than the amount of // data you typically ingest into the delivery stream in 10 seconds. For example, // if you typically ingest data at 1 MB/sec, the value should be 10 MB or higher. SizeInMBs *int64 `min:"1" type:"integer"` } // String returns the string representation func (s ElasticsearchBufferingHints) String() string { return awsutil.Prettify(s) } // Validate inspects the fields of the type to determine if they are valid. func (s *ElasticsearchBufferingHints) Validate() error { invalidParams := aws.ErrInvalidParams{Context: "ElasticsearchBufferingHints"} if s.IntervalInSeconds != nil && *s.IntervalInSeconds < 60 { invalidParams.Add(aws.NewErrParamMinValue("IntervalInSeconds", 60)) } if s.SizeInMBs != nil && *s.SizeInMBs < 1 { invalidParams.Add(aws.NewErrParamMinValue("SizeInMBs", 1)) } if invalidParams.Len() > 0 { return invalidParams } return nil } // Describes the configuration of a destination in Amazon ES. type ElasticsearchDestinationConfiguration struct { _ struct{} `type:"structure"` // The buffering options. If no value is specified, the default values for ElasticsearchBufferingHints // are used. BufferingHints *ElasticsearchBufferingHints `type:"structure"` // The Amazon CloudWatch logging options for your delivery stream. CloudWatchLoggingOptions *CloudWatchLoggingOptions `type:"structure"` // The endpoint to use when communicating with the cluster. Specify either this // ClusterEndpoint or the DomainARN field. ClusterEndpoint *string `min:"1" type:"string"` // The ARN of the Amazon ES domain. The IAM role must have permissions for DescribeElasticsearchDomain, // DescribeElasticsearchDomains, and DescribeElasticsearchDomainConfig after // assuming the role specified in RoleARN. For more information, see Amazon // Resource Names (ARNs) and AWS Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html). // // Specify either ClusterEndpoint or DomainARN. DomainARN *string `min:"1" type:"string"` // The Elasticsearch index name. // // IndexName is a required field IndexName *string `min:"1" type:"string" required:"true"` // The Elasticsearch index rotation period. Index rotation appends a timestamp // to the IndexName to facilitate the expiration of old data. For more information, // see Index Rotation for the Amazon ES Destination (https://docs.aws.amazon.com/firehose/latest/dev/basic-deliver.html#es-index-rotation). // The default value is OneDay. IndexRotationPeriod ElasticsearchIndexRotationPeriod `type:"string" enum:"true"` // The data processing configuration. ProcessingConfiguration *ProcessingConfiguration `type:"structure"` // The retry behavior in case Kinesis Data Firehose is unable to deliver documents // to Amazon ES. The default value is 300 (5 minutes). RetryOptions *ElasticsearchRetryOptions `type:"structure"` // The Amazon Resource Name (ARN) of the IAM role to be assumed by Kinesis Data // Firehose for calling the Amazon ES Configuration API and for indexing documents. // For more information, see Grant Kinesis Data Firehose Access to an Amazon // S3 Destination (https://docs.aws.amazon.com/firehose/latest/dev/controlling-access.html#using-iam-s3) // and Amazon Resource Names (ARNs) and AWS Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html). // // RoleARN is a required field RoleARN *string `min:"1" type:"string" required:"true"` // Defines how documents should be delivered to Amazon S3. When it is set to // FailedDocumentsOnly, Kinesis Data Firehose writes any documents that could // not be indexed to the configured Amazon S3 destination, with elasticsearch-failed/ // appended to the key prefix. When set to AllDocuments, Kinesis Data Firehose // delivers all incoming records to Amazon S3, and also writes failed documents // with elasticsearch-failed/ appended to the prefix. For more information, // see Amazon S3 Backup for the Amazon ES Destination (https://docs.aws.amazon.com/firehose/latest/dev/basic-deliver.html#es-s3-backup). // Default value is FailedDocumentsOnly. S3BackupMode ElasticsearchS3BackupMode `type:"string" enum:"true"` // The configuration for the backup Amazon S3 location. // // S3Configuration is a required field S3Configuration *S3DestinationConfiguration `type:"structure" required:"true"` // The Elasticsearch type name. For Elasticsearch 6.x, there can be only one // type per index. If you try to specify a new type for an existing index that // already has another type, Kinesis Data Firehose returns an error during run // time. // // For Elasticsearch 7.x, don't specify a TypeName. TypeName *string `type:"string"` // The details of the VPC of the Amazon ES destination. VpcConfiguration *VpcConfiguration `type:"structure"` } // String returns the string representation func (s ElasticsearchDestinationConfiguration) String() string { return awsutil.Prettify(s) } // Validate inspects the fields of the type to determine if they are valid. func (s *ElasticsearchDestinationConfiguration) Validate() error { invalidParams := aws.ErrInvalidParams{Context: "ElasticsearchDestinationConfiguration"} if s.ClusterEndpoint != nil && len(*s.ClusterEndpoint) < 1 { invalidParams.Add(aws.NewErrParamMinLen("ClusterEndpoint", 1)) } if s.DomainARN != nil && len(*s.DomainARN) < 1 { invalidParams.Add(aws.NewErrParamMinLen("DomainARN", 1)) } if s.IndexName == nil { invalidParams.Add(aws.NewErrParamRequired("IndexName")) } if s.IndexName != nil && len(*s.IndexName) < 1 { invalidParams.Add(aws.NewErrParamMinLen("IndexName", 1)) } if s.RoleARN == nil { invalidParams.Add(aws.NewErrParamRequired("RoleARN")) } if s.RoleARN != nil && len(*s.RoleARN) < 1 { invalidParams.Add(aws.NewErrParamMinLen("RoleARN", 1)) } if s.S3Configuration == nil { invalidParams.Add(aws.NewErrParamRequired("S3Configuration")) } if s.BufferingHints != nil { if err := s.BufferingHints.Validate(); err != nil { invalidParams.AddNested("BufferingHints", err.(aws.ErrInvalidParams)) } } if s.ProcessingConfiguration != nil { if err := s.ProcessingConfiguration.Validate(); err != nil { invalidParams.AddNested("ProcessingConfiguration", err.(aws.ErrInvalidParams)) } } if s.S3Configuration != nil { if err := s.S3Configuration.Validate(); err != nil { invalidParams.AddNested("S3Configuration", err.(aws.ErrInvalidParams)) } } if s.VpcConfiguration != nil { if err := s.VpcConfiguration.Validate(); err != nil { invalidParams.AddNested("VpcConfiguration", err.(aws.ErrInvalidParams)) } } if invalidParams.Len() > 0 { return invalidParams } return nil } // The destination description in Amazon ES. type ElasticsearchDestinationDescription struct { _ struct{} `type:"structure"` // The buffering options. BufferingHints *ElasticsearchBufferingHints `type:"structure"` // The Amazon CloudWatch logging options. CloudWatchLoggingOptions *CloudWatchLoggingOptions `type:"structure"` // The endpoint to use when communicating with the cluster. Kinesis Data Firehose // uses either this ClusterEndpoint or the DomainARN field to send data to Amazon // ES. ClusterEndpoint *string `min:"1" type:"string"` // The ARN of the Amazon ES domain. For more information, see Amazon Resource // Names (ARNs) and AWS Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html). // // Kinesis Data Firehose uses either ClusterEndpoint or DomainARN to send data // to Amazon ES. DomainARN *string `min:"1" type:"string"` // The Elasticsearch index name. IndexName *string `min:"1" type:"string"` // The Elasticsearch index rotation period IndexRotationPeriod ElasticsearchIndexRotationPeriod `type:"string" enum:"true"` // The data processing configuration. ProcessingConfiguration *ProcessingConfiguration `type:"structure"` // The Amazon ES retry options. RetryOptions *ElasticsearchRetryOptions `type:"structure"` // The Amazon Resource Name (ARN) of the AWS credentials. For more information, // see Amazon Resource Names (ARNs) and AWS Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html). RoleARN *string `min:"1" type:"string"` // The Amazon S3 backup mode. S3BackupMode ElasticsearchS3BackupMode `type:"string" enum:"true"` // The Amazon S3 destination. S3DestinationDescription *S3DestinationDescription `type:"structure"` // The Elasticsearch type name. This applies to Elasticsearch 6.x and lower // versions. For Elasticsearch 7.x, there's no value for TypeName. TypeName *string `type:"string"` // The details of the VPC of the Amazon ES destination. VpcConfigurationDescription *VpcConfigurationDescription `type:"structure"` } // String returns the string representation func (s ElasticsearchDestinationDescription) String() string { return awsutil.Prettify(s) } // Describes an update for a destination in Amazon ES. type ElasticsearchDestinationUpdate struct { _ struct{} `type:"structure"` // The buffering options. If no value is specified, ElasticsearchBufferingHints // object default values are used. BufferingHints *ElasticsearchBufferingHints `type:"structure"` // The CloudWatch logging options for your delivery stream. CloudWatchLoggingOptions *CloudWatchLoggingOptions `type:"structure"` // The endpoint to use when communicating with the cluster. Specify either this // ClusterEndpoint or the DomainARN field. ClusterEndpoint *string `min:"1" type:"string"` // The ARN of the Amazon ES domain. The IAM role must have permissions for DescribeElasticsearchDomain, // DescribeElasticsearchDomains, and DescribeElasticsearchDomainConfig after // assuming the IAM role specified in RoleARN. For more information, see Amazon // Resource Names (ARNs) and AWS Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html). // // Specify either ClusterEndpoint or DomainARN. DomainARN *string `min:"1" type:"string"` // The Elasticsearch index name. IndexName *string `min:"1" type:"string"` // The Elasticsearch index rotation period. Index rotation appends a timestamp // to IndexName to facilitate the expiration of old data. For more information, // see Index Rotation for the Amazon ES Destination (https://docs.aws.amazon.com/firehose/latest/dev/basic-deliver.html#es-index-rotation). // Default value is OneDay. IndexRotationPeriod ElasticsearchIndexRotationPeriod `type:"string" enum:"true"` // The data processing configuration. ProcessingConfiguration *ProcessingConfiguration `type:"structure"` // The retry behavior in case Kinesis Data Firehose is unable to deliver documents // to Amazon ES. The default value is 300 (5 minutes). RetryOptions *ElasticsearchRetryOptions `type:"structure"` // The Amazon Resource Name (ARN) of the IAM role to be assumed by Kinesis Data // Firehose for calling the Amazon ES Configuration API and for indexing documents. // For more information, see Grant Kinesis Data Firehose Access to an Amazon // S3 Destination (https://docs.aws.amazon.com/firehose/latest/dev/controlling-access.html#using-iam-s3) // and Amazon Resource Names (ARNs) and AWS Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html). RoleARN *string `min:"1" type:"string"` // The Amazon S3 destination. S3Update *S3DestinationUpdate `type:"structure"` // The Elasticsearch type name. For Elasticsearch 6.x, there can be only one // type per index. If you try to specify a new type for an existing index that // already has another type, Kinesis Data Firehose returns an error during runtime. // // If you upgrade Elasticsearch from 6.x to 7.x and don’t update your delivery // stream, Kinesis Data Firehose still delivers data to Elasticsearch with the // old index name and type name. If you want to update your delivery stream // with a new index name, provide an empty string for TypeName. TypeName *string `type:"string"` } // String returns the string representation func (s ElasticsearchDestinationUpdate) String() string { return awsutil.Prettify(s) } // Validate inspects the fields of the type to determine if they are valid. func (s *ElasticsearchDestinationUpdate) Validate() error { invalidParams := aws.ErrInvalidParams{Context: "ElasticsearchDestinationUpdate"} if s.ClusterEndpoint != nil && len(*s.ClusterEndpoint) < 1 { invalidParams.Add(aws.NewErrParamMinLen("ClusterEndpoint", 1)) } if s.DomainARN != nil && len(*s.DomainARN) < 1 { invalidParams.Add(aws.NewErrParamMinLen("DomainARN", 1)) } if s.IndexName != nil && len(*s.IndexName) < 1 { invalidParams.Add(aws.NewErrParamMinLen("IndexName", 1)) } if s.RoleARN != nil && len(*s.RoleARN) < 1 { invalidParams.Add(aws.NewErrParamMinLen("RoleARN", 1)) } if s.BufferingHints != nil { if err := s.BufferingHints.Validate(); err != nil { invalidParams.AddNested("BufferingHints", err.(aws.ErrInvalidParams)) } } if s.ProcessingConfiguration != nil { if err := s.ProcessingConfiguration.Validate(); err != nil { invalidParams.AddNested("ProcessingConfiguration", err.(aws.ErrInvalidParams)) } } if s.S3Update != nil { if err := s.S3Update.Validate(); err != nil { invalidParams.AddNested("S3Update", err.(aws.ErrInvalidParams)) } } if invalidParams.Len() > 0 { return invalidParams } return nil } // Configures retry behavior in case Kinesis Data Firehose is unable to deliver // documents to Amazon ES. type ElasticsearchRetryOptions struct { _ struct{} `type:"structure"` // After an initial failure to deliver to Amazon ES, the total amount of time // during which Kinesis Data Firehose retries delivery (including the first // attempt). After this time has elapsed, the failed documents are written to // Amazon S3. Default value is 300 seconds (5 minutes). A value of 0 (zero) // results in no retries. DurationInSeconds *int64 `type:"integer"` } // String returns the string representation func (s ElasticsearchRetryOptions) String() string { return awsutil.Prettify(s) } // Describes the encryption for a destination in Amazon S3. type EncryptionConfiguration struct { _ struct{} `type:"structure"` // The encryption key. KMSEncryptionConfig *KMSEncryptionConfig `type:"structure"` // Specifically override existing encryption information to ensure that no encryption // is used. NoEncryptionConfig NoEncryptionConfig `type:"string" enum:"true"` } // String returns the string representation func (s EncryptionConfiguration) String() string { return awsutil.Prettify(s) } // Validate inspects the fields of the type to determine if they are valid. func (s *EncryptionConfiguration) Validate() error { invalidParams := aws.ErrInvalidParams{Context: "EncryptionConfiguration"} if s.KMSEncryptionConfig != nil { if err := s.KMSEncryptionConfig.Validate(); err != nil { invalidParams.AddNested("KMSEncryptionConfig", err.(aws.ErrInvalidParams)) } } if invalidParams.Len() > 0 { return invalidParams } return nil } // Describes the configuration of a destination in Amazon S3. type ExtendedS3DestinationConfiguration struct { _ struct{} `type:"structure"` // The ARN of the S3 bucket. For more information, see Amazon Resource Names // (ARNs) and AWS Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html). // // BucketARN is a required field BucketARN *string `min:"1" type:"string" required:"true"` // The buffering option. BufferingHints *BufferingHints `type:"structure"` // The Amazon CloudWatch logging options for your delivery stream. CloudWatchLoggingOptions *CloudWatchLoggingOptions `type:"structure"` // The compression format. If no value is specified, the default is UNCOMPRESSED. CompressionFormat CompressionFormat `type:"string" enum:"true"` // The serializer, deserializer, and schema for converting data from the JSON // format to the Parquet or ORC format before writing it to Amazon S3. DataFormatConversionConfiguration *DataFormatConversionConfiguration `type:"structure"` // The encryption configuration. If no value is specified, the default is no // encryption. EncryptionConfiguration *EncryptionConfiguration `type:"structure"` // A prefix that Kinesis Data Firehose evaluates and adds to failed records // before writing them to S3. This prefix appears immediately following the // bucket name. For information about how to specify this prefix, see Custom // Prefixes for Amazon S3 Objects (https://docs.aws.amazon.com/firehose/latest/dev/s3-prefixes.html). ErrorOutputPrefix *string `type:"string"` // The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered // Amazon S3 files. You can also specify a custom prefix, as described in Custom // Prefixes for Amazon S3 Objects (https://docs.aws.amazon.com/firehose/latest/dev/s3-prefixes.html). Prefix *string `type:"string"` // The data processing configuration. ProcessingConfiguration *ProcessingConfiguration `type:"structure"` // The Amazon Resource Name (ARN) of the AWS credentials. For more information, // see Amazon Resource Names (ARNs) and AWS Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html). // // RoleARN is a required field RoleARN *string `min:"1" type:"string" required:"true"` // The configuration for backup in Amazon S3. S3BackupConfiguration *S3DestinationConfiguration `type:"structure"` // The Amazon S3 backup mode. S3BackupMode S3BackupMode `type:"string" enum:"true"` } // String returns the string representation func (s ExtendedS3DestinationConfiguration) String() string { return awsutil.Prettify(s) } // Validate inspects the fields of the type to determine if they are valid. func (s *ExtendedS3DestinationConfiguration) Validate() error { invalidParams := aws.ErrInvalidParams{Context: "ExtendedS3DestinationConfiguration"} if s.BucketARN == nil { invalidParams.Add(aws.NewErrParamRequired("BucketARN")) } if s.BucketARN != nil && len(*s.BucketARN) < 1 { invalidParams.Add(aws.NewErrParamMinLen("BucketARN", 1)) } if s.RoleARN == nil { invalidParams.Add(aws.NewErrParamRequired("RoleARN")) } if s.RoleARN != nil && len(*s.RoleARN) < 1 { invalidParams.Add(aws.NewErrParamMinLen("RoleARN", 1)) } if s.BufferingHints != nil { if err := s.BufferingHints.Validate(); err != nil { invalidParams.AddNested("BufferingHints", err.(aws.ErrInvalidParams)) } } if s.DataFormatConversionConfiguration != nil { if err := s.DataFormatConversionConfiguration.Validate(); err != nil { invalidParams.AddNested("DataFormatConversionConfiguration", err.(aws.ErrInvalidParams)) } } if s.EncryptionConfiguration != nil { if err := s.EncryptionConfiguration.Validate(); err != nil { invalidParams.AddNested("EncryptionConfiguration", err.(aws.ErrInvalidParams)) } } if s.ProcessingConfiguration != nil { if err := s.ProcessingConfiguration.Validate(); err != nil { invalidParams.AddNested("ProcessingConfiguration", err.(aws.ErrInvalidParams)) } } if s.S3BackupConfiguration != nil { if err := s.S3BackupConfiguration.Validate(); err != nil { invalidParams.AddNested("S3BackupConfiguration", err.(aws.ErrInvalidParams)) } } if invalidParams.Len() > 0 { return invalidParams } return nil } // Describes a destination in Amazon S3. type ExtendedS3DestinationDescription struct { _ struct{} `type:"structure"` // The ARN of the S3 bucket. For more information, see Amazon Resource Names // (ARNs) and AWS Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html). // // BucketARN is a required field BucketARN *string `min:"1" type:"string" required:"true"` // The buffering option. // // BufferingHints is a required field BufferingHints *BufferingHints `type:"structure" required:"true"` // The Amazon CloudWatch logging options for your delivery stream. CloudWatchLoggingOptions *CloudWatchLoggingOptions `type:"structure"` // The compression format. If no value is specified, the default is UNCOMPRESSED. // // CompressionFormat is a required field CompressionFormat CompressionFormat `type:"string" required:"true" enum:"true"` // The serializer, deserializer, and schema for converting data from the JSON // format to the Parquet or ORC format before writing it to Amazon S3. DataFormatConversionConfiguration *DataFormatConversionConfiguration `type:"structure"` // The encryption configuration. If no value is specified, the default is no // encryption. // // EncryptionConfiguration is a required field EncryptionConfiguration *EncryptionConfiguration `type:"structure" required:"true"` // A prefix that Kinesis Data Firehose evaluates and adds to failed records // before writing them to S3. This prefix appears immediately following the // bucket name. For information about how to specify this prefix, see Custom // Prefixes for Amazon S3 Objects (https://docs.aws.amazon.com/firehose/latest/dev/s3-prefixes.html). ErrorOutputPrefix *string `type:"string"` // The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered // Amazon S3 files. You can also specify a custom prefix, as described in Custom // Prefixes for Amazon S3 Objects (https://docs.aws.amazon.com/firehose/latest/dev/s3-prefixes.html). Prefix *string `type:"string"` // The data processing configuration. ProcessingConfiguration *ProcessingConfiguration `type:"structure"` // The Amazon Resource Name (ARN) of the AWS credentials. For more information, // see Amazon Resource Names (ARNs) and AWS Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html). // // RoleARN is a required field RoleARN *string `min:"1" type:"string" required:"true"` // The configuration for backup in Amazon S3. S3BackupDescription *S3DestinationDescription `type:"structure"` // The Amazon S3 backup mode. S3BackupMode S3BackupMode `type:"string" enum:"true"` } // String returns the string representation func (s ExtendedS3DestinationDescription) String() string { return awsutil.Prettify(s) } // Describes an update for a destination in Amazon S3. type ExtendedS3DestinationUpdate struct { _ struct{} `type:"structure"` // The ARN of the S3 bucket. For more information, see Amazon Resource Names // (ARNs) and AWS Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html). BucketARN *string `min:"1" type:"string"` // The buffering option. BufferingHints *BufferingHints `type:"structure"` // The Amazon CloudWatch logging options for your delivery stream. CloudWatchLoggingOptions *CloudWatchLoggingOptions `type:"structure"` // The compression format. If no value is specified, the default is UNCOMPRESSED. CompressionFormat CompressionFormat `type:"string" enum:"true"` // The serializer, deserializer, and schema for converting data from the JSON // format to the Parquet or ORC format before writing it to Amazon S3. DataFormatConversionConfiguration *DataFormatConversionConfiguration `type:"structure"` // The encryption configuration. If no value is specified, the default is no // encryption. EncryptionConfiguration *EncryptionConfiguration `type:"structure"` // A prefix that Kinesis Data Firehose evaluates and adds to failed records // before writing them to S3. This prefix appears immediately following the // bucket name. For information about how to specify this prefix, see Custom // Prefixes for Amazon S3 Objects (https://docs.aws.amazon.com/firehose/latest/dev/s3-prefixes.html). ErrorOutputPrefix *string `type:"string"` // The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered // Amazon S3 files. You can also specify a custom prefix, as described in Custom // Prefixes for Amazon S3 Objects (https://docs.aws.amazon.com/firehose/latest/dev/s3-prefixes.html). Prefix *string `type:"string"` // The data processing configuration. ProcessingConfiguration *ProcessingConfiguration `type:"structure"` // The Amazon Resource Name (ARN) of the AWS credentials. For more information, // see Amazon Resource Names (ARNs) and AWS Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html). RoleARN *string `min:"1" type:"string"` // Enables or disables Amazon S3 backup mode. S3BackupMode S3BackupMode `type:"string" enum:"true"` // The Amazon S3 destination for backup. S3BackupUpdate *S3DestinationUpdate `type:"structure"` } // String returns the string representation func (s ExtendedS3DestinationUpdate) String() string { return awsutil.Prettify(s) } // Validate inspects the fields of the type to determine if they are valid. func (s *ExtendedS3DestinationUpdate) Validate() error { invalidParams := aws.ErrInvalidParams{Context: "ExtendedS3DestinationUpdate"} if s.BucketARN != nil && len(*s.BucketARN) < 1 { invalidParams.Add(aws.NewErrParamMinLen("BucketARN", 1)) } if s.RoleARN != nil && len(*s.RoleARN) < 1 { invalidParams.Add(aws.NewErrParamMinLen("RoleARN", 1)) } if s.BufferingHints != nil { if err := s.BufferingHints.Validate(); err != nil { invalidParams.AddNested("BufferingHints", err.(aws.ErrInvalidParams)) } } if s.DataFormatConversionConfiguration != nil { if err := s.DataFormatConversionConfiguration.Validate(); err != nil { invalidParams.AddNested("DataFormatConversionConfiguration", err.(aws.ErrInvalidParams)) } } if s.EncryptionConfiguration != nil { if err := s.EncryptionConfiguration.Validate(); err != nil { invalidParams.AddNested("EncryptionConfiguration", err.(aws.ErrInvalidParams)) } } if s.ProcessingConfiguration != nil { if err := s.ProcessingConfiguration.Validate(); err != nil { invalidParams.AddNested("ProcessingConfiguration", err.(aws.ErrInvalidParams)) } } if s.S3BackupUpdate != nil { if err := s.S3BackupUpdate.Validate(); err != nil { invalidParams.AddNested("S3BackupUpdate", err.(aws.ErrInvalidParams)) } } if invalidParams.Len() > 0 { return invalidParams } return nil } // Provides details in case one of the following operations fails due to an // error related to KMS: CreateDeliveryStream, DeleteDeliveryStream, StartDeliveryStreamEncryption, // StopDeliveryStreamEncryption. type FailureDescription struct { _ struct{} `type:"structure"` // A message providing details about the error that caused the failure. // // Details is a required field Details *string `min:"1" type:"string" required:"true"` // The type of error that caused the failure. // // Type is a required field Type DeliveryStreamFailureType `type:"string" required:"true" enum:"true"` } // String returns the string representation func (s FailureDescription) String() string { return awsutil.Prettify(s) } // The native Hive / HCatalog JsonSerDe. Used by Kinesis Data Firehose for deserializing // data, which means converting it from the JSON format in preparation for serializing // it to the Parquet or ORC format. This is one of two deserializers you can // choose, depending on which one offers the functionality you need. The other // option is the OpenX SerDe. type HiveJsonSerDe struct { _ struct{} `type:"structure"` // Indicates how you want Kinesis Data Firehose to parse the date and timestamps // that may be present in your input data JSON. To specify these format strings, // follow the pattern syntax of JodaTime's DateTimeFormat format strings. For // more information, see Class DateTimeFormat (https://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html). // You can also use the special value millis to parse timestamps in epoch milliseconds. // If you don't specify a format, Kinesis Data Firehose uses java.sql.Timestamp::valueOf // by default. TimestampFormats []string `type:"list"` } // String returns the string representation func (s HiveJsonSerDe) String() string { return awsutil.Prettify(s) } // Specifies the deserializer you want to use to convert the format of the input // data. This parameter is required if Enabled is set to true. type InputFormatConfiguration struct { _ struct{} `type:"structure"` // Specifies which deserializer to use. You can choose either the Apache Hive // JSON SerDe or the OpenX JSON SerDe. If both are non-null, the server rejects // the request. Deserializer *Deserializer `type:"structure"` } // String returns the string representation func (s InputFormatConfiguration) String() string { return awsutil.Prettify(s) } // Describes an encryption key for a destination in Amazon S3. type KMSEncryptionConfig struct { _ struct{} `type:"structure"` // The Amazon Resource Name (ARN) of the encryption key. Must belong to the // same AWS Region as the destination Amazon S3 bucket. For more information, // see Amazon Resource Names (ARNs) and AWS Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html). // // AWSKMSKeyARN is a required field AWSKMSKeyARN *string `min:"1" type:"string" required:"true"` } // String returns the string representation func (s KMSEncryptionConfig) String() string { return awsutil.Prettify(s) } // Validate inspects the fields of the type to determine if they are valid. func (s *KMSEncryptionConfig) Validate() error { invalidParams := aws.ErrInvalidParams{Context: "KMSEncryptionConfig"} if s.AWSKMSKeyARN == nil { invalidParams.Add(aws.NewErrParamRequired("AWSKMSKeyARN")) } if s.AWSKMSKeyARN != nil && len(*s.AWSKMSKeyARN) < 1 { invalidParams.Add(aws.NewErrParamMinLen("AWSKMSKeyARN", 1)) } if invalidParams.Len() > 0 { return invalidParams } return nil } // The stream and role Amazon Resource Names (ARNs) for a Kinesis data stream // used as the source for a delivery stream. type KinesisStreamSourceConfiguration struct { _ struct{} `type:"structure"` // The ARN of the source Kinesis data stream. For more information, see Amazon // Kinesis Data Streams ARN Format (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html#arn-syntax-kinesis-streams). // // KinesisStreamARN is a required field KinesisStreamARN *string `min:"1" type:"string" required:"true"` // The ARN of the role that provides access to the source Kinesis data stream. // For more information, see AWS Identity and Access Management (IAM) ARN Format // (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html#arn-syntax-iam). // // RoleARN is a required field RoleARN *string `min:"1" type:"string" required:"true"` } // String returns the string representation func (s KinesisStreamSourceConfiguration) String() string { return awsutil.Prettify(s) } // Validate inspects the fields of the type to determine if they are valid. func (s *KinesisStreamSourceConfiguration) Validate() error { invalidParams := aws.ErrInvalidParams{Context: "KinesisStreamSourceConfiguration"} if s.KinesisStreamARN == nil { invalidParams.Add(aws.NewErrParamRequired("KinesisStreamARN")) } if s.KinesisStreamARN != nil && len(*s.KinesisStreamARN) < 1 { invalidParams.Add(aws.NewErrParamMinLen("KinesisStreamARN", 1)) } if s.RoleARN == nil { invalidParams.Add(aws.NewErrParamRequired("RoleARN")) } if s.RoleARN != nil && len(*s.RoleARN) < 1 { invalidParams.Add(aws.NewErrParamMinLen("RoleARN", 1)) } if invalidParams.Len() > 0 { return invalidParams } return nil } // Details about a Kinesis data stream used as the source for a Kinesis Data // Firehose delivery stream. type KinesisStreamSourceDescription struct { _ struct{} `type:"structure"` // Kinesis Data Firehose starts retrieving records from the Kinesis data stream // starting with this timestamp. DeliveryStartTimestamp *time.Time `type:"timestamp"` // The Amazon Resource Name (ARN) of the source Kinesis data stream. For more // information, see Amazon Kinesis Data Streams ARN Format (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html#arn-syntax-kinesis-streams). KinesisStreamARN *string `min:"1" type:"string"` // The ARN of the role used by the source Kinesis data stream. For more information, // see AWS Identity and Access Management (IAM) ARN Format (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html#arn-syntax-iam). RoleARN *string `min:"1" type:"string"` } // String returns the string representation func (s KinesisStreamSourceDescription) String() string { return awsutil.Prettify(s) } // The OpenX SerDe. Used by Kinesis Data Firehose for deserializing data, which // means converting it from the JSON format in preparation for serializing it // to the Parquet or ORC format. This is one of two deserializers you can choose, // depending on which one offers the functionality you need. The other option // is the native Hive / HCatalog JsonSerDe. type OpenXJsonSerDe struct { _ struct{} `type:"structure"` // When set to true, which is the default, Kinesis Data Firehose converts JSON // keys to lowercase before deserializing them. CaseInsensitive *bool `type:"boolean"` // Maps column names to JSON keys that aren't identical to the column names. // This is useful when the JSON contains keys that are Hive keywords. For example, // timestamp is a Hive keyword. If you have a JSON key named timestamp, set // this parameter to {"ts": "timestamp"} to map this key to a column named ts. ColumnToJsonKeyMappings map[string]string `type:"map"` // When set to true, specifies that the names of the keys include dots and that // you want Kinesis Data Firehose to replace them with underscores. This is // useful because Apache Hive does not allow dots in column names. For example, // if the JSON contains a key whose name is "a.b", you can define the column // name to be "a_b" when using this option. // // The default is false. ConvertDotsInJsonKeysToUnderscores *bool `type:"boolean"` } // String returns the string representation func (s OpenXJsonSerDe) String() string { return awsutil.Prettify(s) } // A serializer to use for converting data to the ORC format before storing // it in Amazon S3. For more information, see Apache ORC (https://orc.apache.org/docs/). type OrcSerDe struct { _ struct{} `type:"structure"` // The Hadoop Distributed File System (HDFS) block size. This is useful if you // intend to copy the data from Amazon S3 to HDFS before querying. The default // is 256 MiB and the minimum is 64 MiB. Kinesis Data Firehose uses this value // for padding calculations. BlockSizeBytes *int64 `min:"6.7108864e+07" type:"integer"` // The column names for which you want Kinesis Data Firehose to create bloom // filters. The default is null. BloomFilterColumns []string `type:"list"` // The Bloom filter false positive probability (FPP). The lower the FPP, the // bigger the Bloom filter. The default value is 0.05, the minimum is 0, and // the maximum is 1. BloomFilterFalsePositiveProbability *float64 `type:"double"` // The compression code to use over data blocks. The default is SNAPPY. Compression OrcCompression `type:"string" enum:"true"` // Represents the fraction of the total number of non-null rows. To turn off // dictionary encoding, set this fraction to a number that is less than the // number of distinct keys in a dictionary. To always use dictionary encoding, // set this threshold to 1. DictionaryKeyThreshold *float64 `type:"double"` // Set this to true to indicate that you want stripes to be padded to the HDFS // block boundaries. This is useful if you intend to copy the data from Amazon // S3 to HDFS before querying. The default is false. EnablePadding *bool `type:"boolean"` // The version of the file to write. The possible values are V0_11 and V0_12. // The default is V0_12. FormatVersion OrcFormatVersion `type:"string" enum:"true"` // A number between 0 and 1 that defines the tolerance for block padding as // a decimal fraction of stripe size. The default value is 0.05, which means // 5 percent of stripe size. // // For the default values of 64 MiB ORC stripes and 256 MiB HDFS blocks, the // default block padding tolerance of 5 percent reserves a maximum of 3.2 MiB // for padding within the 256 MiB block. In such a case, if the available size // within the block is more than 3.2 MiB, a new, smaller stripe is inserted // to fit within that space. This ensures that no stripe crosses block boundaries // and causes remote reads within a node-local task. // // Kinesis Data Firehose ignores this parameter when OrcSerDe$EnablePadding // is false. PaddingTolerance *float64 `type:"double"` // The number of rows between index entries. The default is 10,000 and the minimum // is 1,000. RowIndexStride *int64 `min:"1000" type:"integer"` // The number of bytes in each stripe. The default is 64 MiB and the minimum // is 8 MiB. StripeSizeBytes *int64 `min:"8.388608e+06" type:"integer"` } // String returns the string representation func (s OrcSerDe) String() string { return awsutil.Prettify(s) } // Validate inspects the fields of the type to determine if they are valid. func (s *OrcSerDe) Validate() error { invalidParams := aws.ErrInvalidParams{Context: "OrcSerDe"} if s.BlockSizeBytes != nil && *s.BlockSizeBytes < 6.7108864e+07 { invalidParams.Add(aws.NewErrParamMinValue("BlockSizeBytes", 6.7108864e+07)) } if s.RowIndexStride != nil && *s.RowIndexStride < 1000 { invalidParams.Add(aws.NewErrParamMinValue("RowIndexStride", 1000)) } if s.StripeSizeBytes != nil && *s.StripeSizeBytes < 8.388608e+06 { invalidParams.Add(aws.NewErrParamMinValue("StripeSizeBytes", 8.388608e+06)) } if invalidParams.Len() > 0 { return invalidParams } return nil } // Specifies the serializer that you want Kinesis Data Firehose to use to convert // the format of your data before it writes it to Amazon S3. This parameter // is required if Enabled is set to true. type OutputFormatConfiguration struct { _ struct{} `type:"structure"` // Specifies which serializer to use. You can choose either the ORC SerDe or // the Parquet SerDe. If both are non-null, the server rejects the request. Serializer *Serializer `type:"structure"` } // String returns the string representation func (s OutputFormatConfiguration) String() string { return awsutil.Prettify(s) } // Validate inspects the fields of the type to determine if they are valid. func (s *OutputFormatConfiguration) Validate() error { invalidParams := aws.ErrInvalidParams{Context: "OutputFormatConfiguration"} if s.Serializer != nil { if err := s.Serializer.Validate(); err != nil { invalidParams.AddNested("Serializer", err.(aws.ErrInvalidParams)) } } if invalidParams.Len() > 0 { return invalidParams } return nil } // A serializer to use for converting data to the Parquet format before storing // it in Amazon S3. For more information, see Apache Parquet (https://parquet.apache.org/documentation/latest/). type ParquetSerDe struct { _ struct{} `type:"structure"` // The Hadoop Distributed File System (HDFS) block size. This is useful if you // intend to copy the data from Amazon S3 to HDFS before querying. The default // is 256 MiB and the minimum is 64 MiB. Kinesis Data Firehose uses this value // for padding calculations. BlockSizeBytes *int64 `min:"6.7108864e+07" type:"integer"` // The compression code to use over data blocks. The possible values are UNCOMPRESSED, // SNAPPY, and GZIP, with the default being SNAPPY. Use SNAPPY for higher decompression // speed. Use GZIP if the compression ratio is more important than speed. Compression ParquetCompression `type:"string" enum:"true"` // Indicates whether to enable dictionary compression. EnableDictionaryCompression *bool `type:"boolean"` // The maximum amount of padding to apply. This is useful if you intend to copy // the data from Amazon S3 to HDFS before querying. The default is 0. MaxPaddingBytes *int64 `type:"integer"` // The Parquet page size. Column chunks are divided into pages. A page is conceptually // an indivisible unit (in terms of compression and encoding). The minimum value // is 64 KiB and the default is 1 MiB. PageSizeBytes *int64 `min:"65536" type:"integer"` // Indicates the version of row format to output. The possible values are V1 // and V2. The default is V1. WriterVersion ParquetWriterVersion `type:"string" enum:"true"` } // String returns the string representation func (s ParquetSerDe) String() string { return awsutil.Prettify(s) } // Validate inspects the fields of the type to determine if they are valid. func (s *ParquetSerDe) Validate() error { invalidParams := aws.ErrInvalidParams{Context: "ParquetSerDe"} if s.BlockSizeBytes != nil && *s.BlockSizeBytes < 6.7108864e+07 { invalidParams.Add(aws.NewErrParamMinValue("BlockSizeBytes", 6.7108864e+07)) } if s.PageSizeBytes != nil && *s.PageSizeBytes < 65536 { invalidParams.Add(aws.NewErrParamMinValue("PageSizeBytes", 65536)) } if invalidParams.Len() > 0 { return invalidParams } return nil } // Describes a data processing configuration. type ProcessingConfiguration struct { _ struct{} `type:"structure"` // Enables or disables data processing. Enabled *bool `type:"boolean"` // The data processors. Processors []Processor `type:"list"` } // String returns the string representation func (s ProcessingConfiguration) String() string { return awsutil.Prettify(s) } // Validate inspects the fields of the type to determine if they are valid. func (s *ProcessingConfiguration) Validate() error { invalidParams := aws.ErrInvalidParams{Context: "ProcessingConfiguration"} if s.Processors != nil { for i, v := range s.Processors { if err := v.Validate(); err != nil { invalidParams.AddNested(fmt.Sprintf("%s[%v]", "Processors", i), err.(aws.ErrInvalidParams)) } } } if invalidParams.Len() > 0 { return invalidParams } return nil } // Describes a data processor. type Processor struct { _ struct{} `type:"structure"` // The processor parameters. Parameters []ProcessorParameter `type:"list"` // The type of processor. // // Type is a required field Type ProcessorType `type:"string" required:"true" enum:"true"` } // String returns the string representation func (s Processor) String() string { return awsutil.Prettify(s) } // Validate inspects the fields of the type to determine if they are valid. func (s *Processor) Validate() error { invalidParams := aws.ErrInvalidParams{Context: "Processor"} if len(s.Type) == 0 { invalidParams.Add(aws.NewErrParamRequired("Type")) } if s.Parameters != nil { for i, v := range s.Parameters { if err := v.Validate(); err != nil { invalidParams.AddNested(fmt.Sprintf("%s[%v]", "Parameters", i), err.(aws.ErrInvalidParams)) } } } if invalidParams.Len() > 0 { return invalidParams } return nil } // Describes the processor parameter. type ProcessorParameter struct { _ struct{} `type:"structure"` // The name of the parameter. // // ParameterName is a required field ParameterName ProcessorParameterName `type:"string" required:"true" enum:"true"` // The parameter value. // // ParameterValue is a required field ParameterValue *string `min:"1" type:"string" required:"true"` } // String returns the string representation func (s ProcessorParameter) String() string { return awsutil.Prettify(s) } // Validate inspects the fields of the type to determine if they are valid. func (s *ProcessorParameter) Validate() error { invalidParams := aws.ErrInvalidParams{Context: "ProcessorParameter"} if len(s.ParameterName) == 0 { invalidParams.Add(aws.NewErrParamRequired("ParameterName")) } if s.ParameterValue == nil { invalidParams.Add(aws.NewErrParamRequired("ParameterValue")) } if s.ParameterValue != nil && len(*s.ParameterValue) < 1 { invalidParams.Add(aws.NewErrParamMinLen("ParameterValue", 1)) } if invalidParams.Len() > 0 { return invalidParams } return nil } // Contains the result for an individual record from a PutRecordBatch request. // If the record is successfully added to your delivery stream, it receives // a record ID. If the record fails to be added to your delivery stream, the // result includes an error code and an error message. type PutRecordBatchResponseEntry struct { _ struct{} `type:"structure"` // The error code for an individual record result. ErrorCode *string `type:"string"` // The error message for an individual record result. ErrorMessage *string `type:"string"` // The ID of the record. RecordId *string `min:"1" type:"string"` } // String returns the string representation func (s PutRecordBatchResponseEntry) String() string { return awsutil.Prettify(s) } // The unit of data in a delivery stream. type Record struct { _ struct{} `type:"structure"` // The data blob, which is base64-encoded when the blob is serialized. The maximum // size of the data blob, before base64-encoding, is 1,000 KiB. // // Data is automatically base64 encoded/decoded by the SDK. // // Data is a required field Data []byte `type:"blob" required:"true"` } // String returns the string representation func (s Record) String() string { return awsutil.Prettify(s) } // Validate inspects the fields of the type to determine if they are valid. func (s *Record) Validate() error { invalidParams := aws.ErrInvalidParams{Context: "Record"} if s.Data == nil { invalidParams.Add(aws.NewErrParamRequired("Data")) } if invalidParams.Len() > 0 { return invalidParams } return nil } // Describes the configuration of a destination in Amazon Redshift. type RedshiftDestinationConfiguration struct { _ struct{} `type:"structure"` // The CloudWatch logging options for your delivery stream. CloudWatchLoggingOptions *CloudWatchLoggingOptions `type:"structure"` // The database connection string. // // ClusterJDBCURL is a required field ClusterJDBCURL *string `min:"1" type:"string" required:"true"` // The COPY command. // // CopyCommand is a required field CopyCommand *CopyCommand `type:"structure" required:"true"` // The user password. // // Password is a required field Password *string `min:"6" type:"string" required:"true" sensitive:"true"` // The data processing configuration. ProcessingConfiguration *ProcessingConfiguration `type:"structure"` // The retry behavior in case Kinesis Data Firehose is unable to deliver documents // to Amazon Redshift. Default value is 3600 (60 minutes). RetryOptions *RedshiftRetryOptions `type:"structure"` // The Amazon Resource Name (ARN) of the AWS credentials. For more information, // see Amazon Resource Names (ARNs) and AWS Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html). // // RoleARN is a required field RoleARN *string `min:"1" type:"string" required:"true"` // The configuration for backup in Amazon S3. S3BackupConfiguration *S3DestinationConfiguration `type:"structure"` // The Amazon S3 backup mode. S3BackupMode RedshiftS3BackupMode `type:"string" enum:"true"` // The configuration for the intermediate Amazon S3 location from which Amazon // Redshift obtains data. Restrictions are described in the topic for CreateDeliveryStream. // // The compression formats SNAPPY or ZIP cannot be specified in RedshiftDestinationConfiguration.S3Configuration // because the Amazon Redshift COPY operation that reads from the S3 bucket // doesn't support these compression formats. // // S3Configuration is a required field S3Configuration *S3DestinationConfiguration `type:"structure" required:"true"` // The name of the user. // // Username is a required field Username *string `min:"1" type:"string" required:"true" sensitive:"true"` } // String returns the string representation func (s RedshiftDestinationConfiguration) String() string { return awsutil.Prettify(s) } // Validate inspects the fields of the type to determine if they are valid. func (s *RedshiftDestinationConfiguration) Validate() error { invalidParams := aws.ErrInvalidParams{Context: "RedshiftDestinationConfiguration"} if s.ClusterJDBCURL == nil { invalidParams.Add(aws.NewErrParamRequired("ClusterJDBCURL")) } if s.ClusterJDBCURL != nil && len(*s.ClusterJDBCURL) < 1 { invalidParams.Add(aws.NewErrParamMinLen("ClusterJDBCURL", 1)) } if s.CopyCommand == nil { invalidParams.Add(aws.NewErrParamRequired("CopyCommand")) } if s.Password == nil { invalidParams.Add(aws.NewErrParamRequired("Password")) } if s.Password != nil && len(*s.Password) < 6 { invalidParams.Add(aws.NewErrParamMinLen("Password", 6)) } if s.RoleARN == nil { invalidParams.Add(aws.NewErrParamRequired("RoleARN")) } if s.RoleARN != nil && len(*s.RoleARN) < 1 { invalidParams.Add(aws.NewErrParamMinLen("RoleARN", 1)) } if s.S3Configuration == nil { invalidParams.Add(aws.NewErrParamRequired("S3Configuration")) } if s.Username == nil { invalidParams.Add(aws.NewErrParamRequired("Username")) } if s.Username != nil && len(*s.Username) < 1 { invalidParams.Add(aws.NewErrParamMinLen("Username", 1)) } if s.CopyCommand != nil { if err := s.CopyCommand.Validate(); err != nil { invalidParams.AddNested("CopyCommand", err.(aws.ErrInvalidParams)) } } if s.ProcessingConfiguration != nil { if err := s.ProcessingConfiguration.Validate(); err != nil { invalidParams.AddNested("ProcessingConfiguration", err.(aws.ErrInvalidParams)) } } if s.S3BackupConfiguration != nil { if err := s.S3BackupConfiguration.Validate(); err != nil { invalidParams.AddNested("S3BackupConfiguration", err.(aws.ErrInvalidParams)) } } if s.S3Configuration != nil { if err := s.S3Configuration.Validate(); err != nil { invalidParams.AddNested("S3Configuration", err.(aws.ErrInvalidParams)) } } if invalidParams.Len() > 0 { return invalidParams } return nil } // Describes a destination in Amazon Redshift. type RedshiftDestinationDescription struct { _ struct{} `type:"structure"` // The Amazon CloudWatch logging options for your delivery stream. CloudWatchLoggingOptions *CloudWatchLoggingOptions `type:"structure"` // The database connection string. // // ClusterJDBCURL is a required field ClusterJDBCURL *string `min:"1" type:"string" required:"true"` // The COPY command. // // CopyCommand is a required field CopyCommand *CopyCommand `type:"structure" required:"true"` // The data processing configuration. ProcessingConfiguration *ProcessingConfiguration `type:"structure"` // The retry behavior in case Kinesis Data Firehose is unable to deliver documents // to Amazon Redshift. Default value is 3600 (60 minutes). RetryOptions *RedshiftRetryOptions `type:"structure"` // The Amazon Resource Name (ARN) of the AWS credentials. For more information, // see Amazon Resource Names (ARNs) and AWS Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html). // // RoleARN is a required field RoleARN *string `min:"1" type:"string" required:"true"` // The configuration for backup in Amazon S3. S3BackupDescription *S3DestinationDescription `type:"structure"` // The Amazon S3 backup mode. S3BackupMode RedshiftS3BackupMode `type:"string" enum:"true"` // The Amazon S3 destination. // // S3DestinationDescription is a required field S3DestinationDescription *S3DestinationDescription `type:"structure" required:"true"` // The name of the user. // // Username is a required field Username *string `min:"1" type:"string" required:"true" sensitive:"true"` } // String returns the string representation func (s RedshiftDestinationDescription) String() string { return awsutil.Prettify(s) } // Describes an update for a destination in Amazon Redshift. type RedshiftDestinationUpdate struct { _ struct{} `type:"structure"` // The Amazon CloudWatch logging options for your delivery stream. CloudWatchLoggingOptions *CloudWatchLoggingOptions `type:"structure"` // The database connection string. ClusterJDBCURL *string `min:"1" type:"string"` // The COPY command. CopyCommand *CopyCommand `type:"structure"` // The user password. Password *string `min:"6" type:"string" sensitive:"true"` // The data processing configuration. ProcessingConfiguration *ProcessingConfiguration `type:"structure"` // The retry behavior in case Kinesis Data Firehose is unable to deliver documents // to Amazon Redshift. Default value is 3600 (60 minutes). RetryOptions *RedshiftRetryOptions `type:"structure"` // The Amazon Resource Name (ARN) of the AWS credentials. For more information, // see Amazon Resource Names (ARNs) and AWS Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html). RoleARN *string `min:"1" type:"string"` // The Amazon S3 backup mode. S3BackupMode RedshiftS3BackupMode `type:"string" enum:"true"` // The Amazon S3 destination for backup. S3BackupUpdate *S3DestinationUpdate `type:"structure"` // The Amazon S3 destination. // // The compression formats SNAPPY or ZIP cannot be specified in RedshiftDestinationUpdate.S3Update // because the Amazon Redshift COPY operation that reads from the S3 bucket // doesn't support these compression formats. S3Update *S3DestinationUpdate `type:"structure"` // The name of the user. Username *string `min:"1" type:"string" sensitive:"true"` } // String returns the string representation func (s RedshiftDestinationUpdate) String() string { return awsutil.Prettify(s) } // Validate inspects the fields of the type to determine if they are valid. func (s *RedshiftDestinationUpdate) Validate() error { invalidParams := aws.ErrInvalidParams{Context: "RedshiftDestinationUpdate"} if s.ClusterJDBCURL != nil && len(*s.ClusterJDBCURL) < 1 { invalidParams.Add(aws.NewErrParamMinLen("ClusterJDBCURL", 1)) } if s.Password != nil && len(*s.Password) < 6 { invalidParams.Add(aws.NewErrParamMinLen("Password", 6)) } if s.RoleARN != nil && len(*s.RoleARN) < 1 { invalidParams.Add(aws.NewErrParamMinLen("RoleARN", 1)) } if s.Username != nil && len(*s.Username) < 1 { invalidParams.Add(aws.NewErrParamMinLen("Username", 1)) } if s.CopyCommand != nil { if err := s.CopyCommand.Validate(); err != nil { invalidParams.AddNested("CopyCommand", err.(aws.ErrInvalidParams)) } } if s.ProcessingConfiguration != nil { if err := s.ProcessingConfiguration.Validate(); err != nil { invalidParams.AddNested("ProcessingConfiguration", err.(aws.ErrInvalidParams)) } } if s.S3BackupUpdate != nil { if err := s.S3BackupUpdate.Validate(); err != nil { invalidParams.AddNested("S3BackupUpdate", err.(aws.ErrInvalidParams)) } } if s.S3Update != nil { if err := s.S3Update.Validate(); err != nil { invalidParams.AddNested("S3Update", err.(aws.ErrInvalidParams)) } } if invalidParams.Len() > 0 { return invalidParams } return nil } // Configures retry behavior in case Kinesis Data Firehose is unable to deliver // documents to Amazon Redshift. type RedshiftRetryOptions struct { _ struct{} `type:"structure"` // The length of time during which Kinesis Data Firehose retries delivery after // a failure, starting from the initial request and including the first attempt. // The default value is 3600 seconds (60 minutes). Kinesis Data Firehose does // not retry if the value of DurationInSeconds is 0 (zero) or if the first delivery // attempt takes longer than the current value. DurationInSeconds *int64 `type:"integer"` } // String returns the string representation func (s RedshiftRetryOptions) String() string { return awsutil.Prettify(s) } // Describes the configuration of a destination in Amazon S3. type S3DestinationConfiguration struct { _ struct{} `type:"structure"` // The ARN of the S3 bucket. For more information, see Amazon Resource Names // (ARNs) and AWS Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html). // // BucketARN is a required field BucketARN *string `min:"1" type:"string" required:"true"` // The buffering option. If no value is specified, BufferingHints object default // values are used. BufferingHints *BufferingHints `type:"structure"` // The CloudWatch logging options for your delivery stream. CloudWatchLoggingOptions *CloudWatchLoggingOptions `type:"structure"` // The compression format. If no value is specified, the default is UNCOMPRESSED. // // The compression formats SNAPPY or ZIP cannot be specified for Amazon Redshift // destinations because they are not supported by the Amazon Redshift COPY operation // that reads from the S3 bucket. CompressionFormat CompressionFormat `type:"string" enum:"true"` // The encryption configuration. If no value is specified, the default is no // encryption. EncryptionConfiguration *EncryptionConfiguration `type:"structure"` // A prefix that Kinesis Data Firehose evaluates and adds to failed records // before writing them to S3. This prefix appears immediately following the // bucket name. For information about how to specify this prefix, see Custom // Prefixes for Amazon S3 Objects (https://docs.aws.amazon.com/firehose/latest/dev/s3-prefixes.html). ErrorOutputPrefix *string `type:"string"` // The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered // Amazon S3 files. You can also specify a custom prefix, as described in Custom // Prefixes for Amazon S3 Objects (https://docs.aws.amazon.com/firehose/latest/dev/s3-prefixes.html). Prefix *string `type:"string"` // The Amazon Resource Name (ARN) of the AWS credentials. For more information, // see Amazon Resource Names (ARNs) and AWS Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html). // // RoleARN is a required field RoleARN *string `min:"1" type:"string" required:"true"` } // String returns the string representation func (s S3DestinationConfiguration) String() string { return awsutil.Prettify(s) } // Validate inspects the fields of the type to determine if they are valid. func (s *S3DestinationConfiguration) Validate() error { invalidParams := aws.ErrInvalidParams{Context: "S3DestinationConfiguration"} if s.BucketARN == nil { invalidParams.Add(aws.NewErrParamRequired("BucketARN")) } if s.BucketARN != nil && len(*s.BucketARN) < 1 { invalidParams.Add(aws.NewErrParamMinLen("BucketARN", 1)) } if s.RoleARN == nil { invalidParams.Add(aws.NewErrParamRequired("RoleARN")) } if s.RoleARN != nil && len(*s.RoleARN) < 1 { invalidParams.Add(aws.NewErrParamMinLen("RoleARN", 1)) } if s.BufferingHints != nil { if err := s.BufferingHints.Validate(); err != nil { invalidParams.AddNested("BufferingHints", err.(aws.ErrInvalidParams)) } } if s.EncryptionConfiguration != nil { if err := s.EncryptionConfiguration.Validate(); err != nil { invalidParams.AddNested("EncryptionConfiguration", err.(aws.ErrInvalidParams)) } } if invalidParams.Len() > 0 { return invalidParams } return nil } // Describes a destination in Amazon S3. type S3DestinationDescription struct { _ struct{} `type:"structure"` // The ARN of the S3 bucket. For more information, see Amazon Resource Names // (ARNs) and AWS Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html). // // BucketARN is a required field BucketARN *string `min:"1" type:"string" required:"true"` // The buffering option. If no value is specified, BufferingHints object default // values are used. // // BufferingHints is a required field BufferingHints *BufferingHints `type:"structure" required:"true"` // The Amazon CloudWatch logging options for your delivery stream. CloudWatchLoggingOptions *CloudWatchLoggingOptions `type:"structure"` // The compression format. If no value is specified, the default is UNCOMPRESSED. // // CompressionFormat is a required field CompressionFormat CompressionFormat `type:"string" required:"true" enum:"true"` // The encryption configuration. If no value is specified, the default is no // encryption. // // EncryptionConfiguration is a required field EncryptionConfiguration *EncryptionConfiguration `type:"structure" required:"true"` // A prefix that Kinesis Data Firehose evaluates and adds to failed records // before writing them to S3. This prefix appears immediately following the // bucket name. For information about how to specify this prefix, see Custom // Prefixes for Amazon S3 Objects (https://docs.aws.amazon.com/firehose/latest/dev/s3-prefixes.html). ErrorOutputPrefix *string `type:"string"` // The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered // Amazon S3 files. You can also specify a custom prefix, as described in Custom // Prefixes for Amazon S3 Objects (https://docs.aws.amazon.com/firehose/latest/dev/s3-prefixes.html). Prefix *string `type:"string"` // The Amazon Resource Name (ARN) of the AWS credentials. For more information, // see Amazon Resource Names (ARNs) and AWS Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html). // // RoleARN is a required field RoleARN *string `min:"1" type:"string" required:"true"` } // String returns the string representation func (s S3DestinationDescription) String() string { return awsutil.Prettify(s) } // Describes an update for a destination in Amazon S3. type S3DestinationUpdate struct { _ struct{} `type:"structure"` // The ARN of the S3 bucket. For more information, see Amazon Resource Names // (ARNs) and AWS Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html). BucketARN *string `min:"1" type:"string"` // The buffering option. If no value is specified, BufferingHints object default // values are used. BufferingHints *BufferingHints `type:"structure"` // The CloudWatch logging options for your delivery stream. CloudWatchLoggingOptions *CloudWatchLoggingOptions `type:"structure"` // The compression format. If no value is specified, the default is UNCOMPRESSED. // // The compression formats SNAPPY or ZIP cannot be specified for Amazon Redshift // destinations because they are not supported by the Amazon Redshift COPY operation // that reads from the S3 bucket. CompressionFormat CompressionFormat `type:"string" enum:"true"` // The encryption configuration. If no value is specified, the default is no // encryption. EncryptionConfiguration *EncryptionConfiguration `type:"structure"` // A prefix that Kinesis Data Firehose evaluates and adds to failed records // before writing them to S3. This prefix appears immediately following the // bucket name. For information about how to specify this prefix, see Custom // Prefixes for Amazon S3 Objects (https://docs.aws.amazon.com/firehose/latest/dev/s3-prefixes.html). ErrorOutputPrefix *string `type:"string"` // The "YYYY/MM/DD/HH" time format prefix is automatically used for delivered // Amazon S3 files. You can also specify a custom prefix, as described in Custom // Prefixes for Amazon S3 Objects (https://docs.aws.amazon.com/firehose/latest/dev/s3-prefixes.html). Prefix *string `type:"string"` // The Amazon Resource Name (ARN) of the AWS credentials. For more information, // see Amazon Resource Names (ARNs) and AWS Service Namespaces (https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html). RoleARN *string `min:"1" type:"string"` } // String returns the string representation func (s S3DestinationUpdate) String() string { return awsutil.Prettify(s) } // Validate inspects the fields of the type to determine if they are valid. func (s *S3DestinationUpdate) Validate() error { invalidParams := aws.ErrInvalidParams{Context: "S3DestinationUpdate"} if s.BucketARN != nil && len(*s.BucketARN) < 1 { invalidParams.Add(aws.NewErrParamMinLen("BucketARN", 1)) } if s.RoleARN != nil && len(*s.RoleARN) < 1 { invalidParams.Add(aws.NewErrParamMinLen("RoleARN", 1)) } if s.BufferingHints != nil { if err := s.BufferingHints.Validate(); err != nil { invalidParams.AddNested("BufferingHints", err.(aws.ErrInvalidParams)) } } if s.EncryptionConfiguration != nil { if err := s.EncryptionConfiguration.Validate(); err != nil { invalidParams.AddNested("EncryptionConfiguration", err.(aws.ErrInvalidParams)) } } if invalidParams.Len() > 0 { return invalidParams } return nil } // Specifies the schema to which you want Kinesis Data Firehose to configure // your data before it writes it to Amazon S3. This parameter is required if // Enabled is set to true. type SchemaConfiguration struct { _ struct{} `type:"structure"` // The ID of the AWS Glue Data Catalog. If you don't supply this, the AWS account // ID is used by default. CatalogId *string `min:"1" type:"string"` // Specifies the name of the AWS Glue database that contains the schema for // the output data. DatabaseName *string `min:"1" type:"string"` // If you don't specify an AWS Region, the default is the current Region. Region *string `min:"1" type:"string"` // The role that Kinesis Data Firehose can use to access AWS Glue. This role // must be in the same account you use for Kinesis Data Firehose. Cross-account // roles aren't allowed. RoleARN *string `min:"1" type:"string"` // Specifies the AWS Glue table that contains the column information that constitutes // your data schema. TableName *string `min:"1" type:"string"` // Specifies the table version for the output data schema. If you don't specify // this version ID, or if you set it to LATEST, Kinesis Data Firehose uses the // most recent version. This means that any updates to the table are automatically // picked up. VersionId *string `min:"1" type:"string"` } // String returns the string representation func (s SchemaConfiguration) String() string { return awsutil.Prettify(s) } // Validate inspects the fields of the type to determine if they are valid. func (s *SchemaConfiguration) Validate() error { invalidParams := aws.ErrInvalidParams{Context: "SchemaConfiguration"} if s.CatalogId != nil && len(*s.CatalogId) < 1 { invalidParams.Add(aws.NewErrParamMinLen("CatalogId", 1)) } if s.DatabaseName != nil && len(*s.DatabaseName) < 1 { invalidParams.Add(aws.NewErrParamMinLen("DatabaseName", 1)) } if s.Region != nil && len(*s.Region) < 1 { invalidParams.Add(aws.NewErrParamMinLen("Region", 1)) } if s.RoleARN != nil && len(*s.RoleARN) < 1 { invalidParams.Add(aws.NewErrParamMinLen("RoleARN", 1)) } if s.TableName != nil && len(*s.TableName) < 1 { invalidParams.Add(aws.NewErrParamMinLen("TableName", 1)) } if s.VersionId != nil && len(*s.VersionId) < 1 { invalidParams.Add(aws.NewErrParamMinLen("VersionId", 1)) } if invalidParams.Len() > 0 { return invalidParams } return nil } // The serializer that you want Kinesis Data Firehose to use to convert data // to the target format before writing it to Amazon S3. Kinesis Data Firehose // supports two types of serializers: the ORC SerDe (https://hive.apache.org/javadocs/r1.2.2/api/org/apache/hadoop/hive/ql/io/orc/OrcSerde.html) // and the Parquet SerDe (https://hive.apache.org/javadocs/r1.2.2/api/org/apache/hadoop/hive/ql/io/parquet/serde/ParquetHiveSerDe.html). type Serializer struct { _ struct{} `type:"structure"` // A serializer to use for converting data to the ORC format before storing // it in Amazon S3. For more information, see Apache ORC (https://orc.apache.org/docs/). OrcSerDe *OrcSerDe `type:"structure"` // A serializer to use for converting data to the Parquet format before storing // it in Amazon S3. For more information, see Apache Parquet (https://parquet.apache.org/documentation/latest/). ParquetSerDe *ParquetSerDe `type:"structure"` } // String returns the string representation func (s Serializer) String() string { return awsutil.Prettify(s) } // Validate inspects the fields of the type to determine if they are valid. func (s *Serializer) Validate() error { invalidParams := aws.ErrInvalidParams{Context: "Serializer"} if s.OrcSerDe != nil { if err := s.OrcSerDe.Validate(); err != nil { invalidParams.AddNested("OrcSerDe", err.(aws.ErrInvalidParams)) } } if s.ParquetSerDe != nil { if err := s.ParquetSerDe.Validate(); err != nil { invalidParams.AddNested("ParquetSerDe", err.(aws.ErrInvalidParams)) } } if invalidParams.Len() > 0 { return invalidParams } return nil } // Details about a Kinesis data stream used as the source for a Kinesis Data // Firehose delivery stream. type SourceDescription struct { _ struct{} `type:"structure"` // The KinesisStreamSourceDescription value for the source Kinesis data stream. KinesisStreamSourceDescription *KinesisStreamSourceDescription `type:"structure"` } // String returns the string representation func (s SourceDescription) String() string { return awsutil.Prettify(s) } // Describes the configuration of a destination in Splunk. type SplunkDestinationConfiguration struct { _ struct{} `type:"structure"` // The Amazon CloudWatch logging options for your delivery stream. CloudWatchLoggingOptions *CloudWatchLoggingOptions `type:"structure"` // The amount of time that Kinesis Data Firehose waits to receive an acknowledgment // from Splunk after it sends it data. At the end of the timeout period, Kinesis // Data Firehose either tries to send the data again or considers it an error, // based on your retry settings. HECAcknowledgmentTimeoutInSeconds *int64 `min:"180" type:"integer"` // The HTTP Event Collector (HEC) endpoint to which Kinesis Data Firehose sends // your data. // // HECEndpoint is a required field HECEndpoint *string `type:"string" required:"true"` // This type can be either "Raw" or "Event." // // HECEndpointType is a required field HECEndpointType HECEndpointType `type:"string" required:"true" enum:"true"` // This is a GUID that you obtain from your Splunk cluster when you create a // new HEC endpoint. // // HECToken is a required field HECToken *string `type:"string" required:"true"` // The data processing configuration. ProcessingConfiguration *ProcessingConfiguration `type:"structure"` // The retry behavior in case Kinesis Data Firehose is unable to deliver data // to Splunk, or if it doesn't receive an acknowledgment of receipt from Splunk. RetryOptions *SplunkRetryOptions `type:"structure"` // Defines how documents should be delivered to Amazon S3. When set to FailedDocumentsOnly, // Kinesis Data Firehose writes any data that could not be indexed to the configured // Amazon S3 destination. When set to AllDocuments, Kinesis Data Firehose delivers // all incoming records to Amazon S3, and also writes failed documents to Amazon // S3. Default value is FailedDocumentsOnly. S3BackupMode SplunkS3BackupMode `type:"string" enum:"true"` // The configuration for the backup Amazon S3 location. // // S3Configuration is a required field S3Configuration *S3DestinationConfiguration `type:"structure" required:"true"` } // String returns the string representation func (s SplunkDestinationConfiguration) String() string { return awsutil.Prettify(s) } // Validate inspects the fields of the type to determine if they are valid. func (s *SplunkDestinationConfiguration) Validate() error { invalidParams := aws.ErrInvalidParams{Context: "SplunkDestinationConfiguration"} if s.HECAcknowledgmentTimeoutInSeconds != nil && *s.HECAcknowledgmentTimeoutInSeconds < 180 { invalidParams.Add(aws.NewErrParamMinValue("HECAcknowledgmentTimeoutInSeconds", 180)) } if s.HECEndpoint == nil { invalidParams.Add(aws.NewErrParamRequired("HECEndpoint")) } if len(s.HECEndpointType) == 0 { invalidParams.Add(aws.NewErrParamRequired("HECEndpointType")) } if s.HECToken == nil { invalidParams.Add(aws.NewErrParamRequired("HECToken")) } if s.S3Configuration == nil { invalidParams.Add(aws.NewErrParamRequired("S3Configuration")) } if s.ProcessingConfiguration != nil { if err := s.ProcessingConfiguration.Validate(); err != nil { invalidParams.AddNested("ProcessingConfiguration", err.(aws.ErrInvalidParams)) } } if s.S3Configuration != nil { if err := s.S3Configuration.Validate(); err != nil { invalidParams.AddNested("S3Configuration", err.(aws.ErrInvalidParams)) } } if invalidParams.Len() > 0 { return invalidParams } return nil } // Describes a destination in Splunk. type SplunkDestinationDescription struct { _ struct{} `type:"structure"` // The Amazon CloudWatch logging options for your delivery stream. CloudWatchLoggingOptions *CloudWatchLoggingOptions `type:"structure"` // The amount of time that Kinesis Data Firehose waits to receive an acknowledgment // from Splunk after it sends it data. At the end of the timeout period, Kinesis // Data Firehose either tries to send the data again or considers it an error, // based on your retry settings. HECAcknowledgmentTimeoutInSeconds *int64 `min:"180" type:"integer"` // The HTTP Event Collector (HEC) endpoint to which Kinesis Data Firehose sends // your data. HECEndpoint *string `type:"string"` // This type can be either "Raw" or "Event." HECEndpointType HECEndpointType `type:"string" enum:"true"` // A GUID you obtain from your Splunk cluster when you create a new HEC endpoint. HECToken *string `type:"string"` // The data processing configuration. ProcessingConfiguration *ProcessingConfiguration `type:"structure"` // The retry behavior in case Kinesis Data Firehose is unable to deliver data // to Splunk or if it doesn't receive an acknowledgment of receipt from Splunk. RetryOptions *SplunkRetryOptions `type:"structure"` // Defines how documents should be delivered to Amazon S3. When set to FailedDocumentsOnly, // Kinesis Data Firehose writes any data that could not be indexed to the configured // Amazon S3 destination. When set to AllDocuments, Kinesis Data Firehose delivers // all incoming records to Amazon S3, and also writes failed documents to Amazon // S3. Default value is FailedDocumentsOnly. S3BackupMode SplunkS3BackupMode `type:"string" enum:"true"` // The Amazon S3 destination.> S3DestinationDescription *S3DestinationDescription `type:"structure"` } // String returns the string representation func (s SplunkDestinationDescription) String() string { return awsutil.Prettify(s) } // Describes an update for a destination in Splunk. type SplunkDestinationUpdate struct { _ struct{} `type:"structure"` // The Amazon CloudWatch logging options for your delivery stream. CloudWatchLoggingOptions *CloudWatchLoggingOptions `type:"structure"` // The amount of time that Kinesis Data Firehose waits to receive an acknowledgment // from Splunk after it sends data. At the end of the timeout period, Kinesis // Data Firehose either tries to send the data again or considers it an error, // based on your retry settings. HECAcknowledgmentTimeoutInSeconds *int64 `min:"180" type:"integer"` // The HTTP Event Collector (HEC) endpoint to which Kinesis Data Firehose sends // your data. HECEndpoint *string `type:"string"` // This type can be either "Raw" or "Event." HECEndpointType HECEndpointType `type:"string" enum:"true"` // A GUID that you obtain from your Splunk cluster when you create a new HEC // endpoint. HECToken *string `type:"string"` // The data processing configuration. ProcessingConfiguration *ProcessingConfiguration `type:"structure"` // The retry behavior in case Kinesis Data Firehose is unable to deliver data // to Splunk or if it doesn't receive an acknowledgment of receipt from Splunk. RetryOptions *SplunkRetryOptions `type:"structure"` // Defines how documents should be delivered to Amazon S3. When set to FailedDocumentsOnly, // Kinesis Data Firehose writes any data that could not be indexed to the configured // Amazon S3 destination. When set to AllDocuments, Kinesis Data Firehose delivers // all incoming records to Amazon S3, and also writes failed documents to Amazon // S3. Default value is FailedDocumentsOnly. S3BackupMode SplunkS3BackupMode `type:"string" enum:"true"` // Your update to the configuration of the backup Amazon S3 location. S3Update *S3DestinationUpdate `type:"structure"` } // String returns the string representation func (s SplunkDestinationUpdate) String() string { return awsutil.Prettify(s) } // Validate inspects the fields of the type to determine if they are valid. func (s *SplunkDestinationUpdate) Validate() error { invalidParams := aws.ErrInvalidParams{Context: "SplunkDestinationUpdate"} if s.HECAcknowledgmentTimeoutInSeconds != nil && *s.HECAcknowledgmentTimeoutInSeconds < 180 { invalidParams.Add(aws.NewErrParamMinValue("HECAcknowledgmentTimeoutInSeconds", 180)) } if s.ProcessingConfiguration != nil { if err := s.ProcessingConfiguration.Validate(); err != nil { invalidParams.AddNested("ProcessingConfiguration", err.(aws.ErrInvalidParams)) } } if s.S3Update != nil { if err := s.S3Update.Validate(); err != nil { invalidParams.AddNested("S3Update", err.(aws.ErrInvalidParams)) } } if invalidParams.Len() > 0 { return invalidParams } return nil } // Configures retry behavior in case Kinesis Data Firehose is unable to deliver // documents to Splunk, or if it doesn't receive an acknowledgment from Splunk. type SplunkRetryOptions struct { _ struct{} `type:"structure"` // The total amount of time that Kinesis Data Firehose spends on retries. This // duration starts after the initial attempt to send data to Splunk fails. It // doesn't include the periods during which Kinesis Data Firehose waits for // acknowledgment from Splunk after each attempt. DurationInSeconds *int64 `type:"integer"` } // String returns the string representation func (s SplunkRetryOptions) String() string { return awsutil.Prettify(s) } // Metadata that you can assign to a delivery stream, consisting of a key-value // pair. type Tag struct { _ struct{} `type:"structure"` // A unique identifier for the tag. Maximum length: 128 characters. Valid characters: // Unicode letters, digits, white space, _ . / = + - % @ // // Key is a required field Key *string `min:"1" type:"string" required:"true"` // An optional string, which you can use to describe or define the tag. Maximum // length: 256 characters. Valid characters: Unicode letters, digits, white // space, _ . / = + - % @ Value *string `type:"string"` } // String returns the string representation func (s Tag) String() string { return awsutil.Prettify(s) } // Validate inspects the fields of the type to determine if they are valid. func (s *Tag) Validate() error { invalidParams := aws.ErrInvalidParams{Context: "Tag"} if s.Key == nil { invalidParams.Add(aws.NewErrParamRequired("Key")) } if s.Key != nil && len(*s.Key) < 1 { invalidParams.Add(aws.NewErrParamMinLen("Key", 1)) } if invalidParams.Len() > 0 { return invalidParams } return nil } // The details of the VPC of the Amazon ES destination. type VpcConfiguration struct { _ struct{} `type:"structure"` // The ARN of the IAM role that you want the delivery stream to use to create // endpoints in the destination VPC. // // RoleARN is a required field RoleARN *string `min:"1" type:"string" required:"true"` // The IDs of the security groups that you want Kinesis Data Firehose to use // when it creates ENIs in the VPC of the Amazon ES destination. // // SecurityGroupIds is a required field SecurityGroupIds []string `min:"1" type:"list" required:"true"` // The IDs of the subnets that you want Kinesis Data Firehose to use to create // ENIs in the VPC of the Amazon ES destination. Make sure that the routing // tables and inbound and outbound rules allow traffic to flow from the subnets // whose IDs are specified here to the subnets that have the destination Amazon // ES endpoints. Kinesis Data Firehose creates at least one ENI in each of the // subnets that are specified here. Do not delete or modify these ENIs. // // The number of ENIs that Kinesis Data Firehose creates in the subnets specified // here scales up and down automatically based on throughput. To enable Kinesis // Data Firehose to scale up the number of ENIs to match throughput, ensure // that you have sufficient quota. To help you calculate the quota you need, // assume that Kinesis Data Firehose can create up to three ENIs for this delivery // stream for each of the subnets specified here. For more information about // ENI quota, see Network Interfaces (https://docs.aws.amazon.com/vpc/latest/userguide/amazon-vpc-limits.html#vpc-limits-enis) // in the Amazon VPC Quotas topic. // // SubnetIds is a required field SubnetIds []string `min:"1" type:"list" required:"true"` } // String returns the string representation func (s VpcConfiguration) String() string { return awsutil.Prettify(s) } // Validate inspects the fields of the type to determine if they are valid. func (s *VpcConfiguration) Validate() error { invalidParams := aws.ErrInvalidParams{Context: "VpcConfiguration"} if s.RoleARN == nil { invalidParams.Add(aws.NewErrParamRequired("RoleARN")) } if s.RoleARN != nil && len(*s.RoleARN) < 1 { invalidParams.Add(aws.NewErrParamMinLen("RoleARN", 1)) } if s.SecurityGroupIds == nil { invalidParams.Add(aws.NewErrParamRequired("SecurityGroupIds")) } if s.SecurityGroupIds != nil && len(s.SecurityGroupIds) < 1 { invalidParams.Add(aws.NewErrParamMinLen("SecurityGroupIds", 1)) } if s.SubnetIds == nil { invalidParams.Add(aws.NewErrParamRequired("SubnetIds")) } if s.SubnetIds != nil && len(s.SubnetIds) < 1 { invalidParams.Add(aws.NewErrParamMinLen("SubnetIds", 1)) } if invalidParams.Len() > 0 { return invalidParams } return nil } // The details of the VPC of the Amazon ES destination. type VpcConfigurationDescription struct { _ struct{} `type:"structure"` // The ARN of the IAM role that you want the delivery stream uses to create // endpoints in the destination VPC. // // RoleARN is a required field RoleARN *string `min:"1" type:"string" required:"true"` // The IDs of the security groups that Kinesis Data Firehose uses when it creates // ENIs in the VPC of the Amazon ES destination. // // SecurityGroupIds is a required field SecurityGroupIds []string `min:"1" type:"list" required:"true"` // The IDs of the subnets that Kinesis Data Firehose uses to create ENIs in // the VPC of the Amazon ES destination. Make sure that the routing tables and // inbound and outbound rules allow traffic to flow from the subnets whose IDs // are specified here to the subnets that have the destination Amazon ES endpoints. // Kinesis Data Firehose creates at least one ENI in each of the subnets that // are specified here. Do not delete or modify these ENIs. // // The number of ENIs that Kinesis Data Firehose creates in the subnets specified // here scales up and down automatically based on throughput. To enable Kinesis // Data Firehose to scale up the number of ENIs to match throughput, ensure // that you have sufficient quota. To help you calculate the quota you need, // assume that Kinesis Data Firehose can create up to three ENIs for this delivery // stream for each of the subnets specified here. For more information about // ENI quota, see Network Interfaces (https://docs.aws.amazon.com/vpc/latest/userguide/amazon-vpc-limits.html#vpc-limits-enis) // in the Amazon VPC Quotas topic. // // SubnetIds is a required field SubnetIds []string `min:"1" type:"list" required:"true"` // The ID of the Amazon ES destination's VPC. // // VpcId is a required field VpcId *string `min:"1" type:"string" required:"true"` } // String returns the string representation func (s VpcConfigurationDescription) String() string { return awsutil.Prettify(s) }