Clikhouse + Amazon SNS通知

如何解决Clikhouse + Amazon SNS通知

我尝试将Amazon SNS通知eventype = Open插入ClickHouse，Json模式很复杂，所以我不怎么创建表（嵌套在嵌套...中）

{
  "eventType":"Open","mail":{
    "commonHeaders":{
      "from":[
        "sender@example.com"
      ],"messageId":"EXAMPLE7c191be45-e9aedb9a-02f9-4d12-a87d-dd0099a07f8a-000000","subject":"Message sent from Amazon SES","to":[
        "recipient@example.com"
      ]
    },"destination":[
      "recipient@example.com"
    ],"headers ":[
      {
        "name":"X-SES-CONfigURATION-SET","value":"ConfigSet"
      },{
        "name":"X-SES-MESSAGE-TAGS","value":"myCustomTag1=myCustomValue1,myCustomTag2=myCustomValue2"
      },{
        "name":"From","value":"sender@example.com"
      },{
        "name":"To","value":"recipient@example.com"
      },{
        "name":"Subject","value":"Message sent from Amazon SES"
      },{
        "name":"MIME-Version","value":"1.0"
      },{
        "name":"Content-Type","value":"multipart/alternative; boundary=\"XBoundary\""
      }
    ],"headersTruncated":false,"sendingAccountId":"123456789012","source":"sender@example.com","tags":{
      "myCustomTag1":[
        "myCustomValue1"
      ],"myCustomTag2":[
        "myCustomValue2"
      ],"ses:caller-identity":[
        "ses-user"
      ],"ses:configuration-set":[
        "ConfigSet"
      ],"ses:from-domain":[
        "example.com"
      ],"ses:source-ip":[
        "192.0.2.0"
      ]
    },"timestamp":"2017-08-09T21:59:49.927Z"
  },"open":{
    "ipAddress":"192.0.2.1","timestamp":"2017-08-09T22:00:19.652Z","userAgent":"Mozilla/5.0 (iPhone; cpu iPhone OS 10_3_3 like Mac OS X) AppleWebKit/603.3.8 (KHTML,like Gecko) Mobile/14G60"
  }
}

我尝试了INSERT INTO Open FORMAT JSONEachRow和INSERT INTO Open FORMAT JSONCompact，但是没有用。

谢谢。

解决方法

您应将JSON转换为更简单的形式而不嵌套，并使用JSONEachRow。

或将数据作为JSONAsString插入CH并使用JSONExtract进行转换

create table i(J String) Engine=Null;
create table f(a String,i Int64,f Float64) Engine=MergeTree order by a;

create materialized view vv to f
as select (JSONExtract(J,'Tuple(String,Tuple(Int64,Float64))') as x),x.1 as a,x.2.1 as i,x.2.2 as f
from i;

echo '{"s": "val1","b2": {"i": 42,"f": 0.1}}' |clickhouse-client -q "insert into i format JSONAsString"

select * from f
┌─a────┬──i─┬───f─┐
│ val1 │ 42 │ 0.1 │
└──────┴────┴─────┘