如何解决AWS，dotnet spark和redshift不起作用
嗨，我在使Redshift和dotnet spark工作方面遇到问题：
此配置用于使它在调试模式下工作：
C:\bin\spark-2.4.1-bin-hadoop2.7\bin\spark-submit.cmd `
--jars spark-redshift_2.11-2.0.1.jar,aws-java-sdk-1.11.869.jar,aws-java-sdk-s3-1.11.832.jar,hadoop-aws-3.3.0.jar `
--class org.apache.spark.deploy.dotnet.DotnetRunner `
--master local `
--packages org.apache.spark:spark-avro_2.11:2.4.0 `
microsoft-spark-2.4.x-0.12.1.jar `
debug
这是我的代码摘录：
// Create a Spark session.
SparkSession spark = SparkSession
.Builder()
.AppName("DataScraping ETL into Redshift")
//.Config("spark.hadoop.fs.s3.access.key",awsAccessKey)
//.Config("spark.hadoop.fs.s3.awsSecretAccessKey",awsSecretKey)
.GetorCreate();
var productState = spark.Read()
//.Format("com.databricks.spark.redshift")
//.Jdbc(url,table,new Dictionary<string,string>
//{
// ["temporary_aws_access_key_id"] = awsAccessKey,// ["temporary_aws_secret_access_key"] = awsSecretKey,// ["aws_iam_role"] = iam_role,// ["tempdir"] = s3Bucket,// ["jdbcdriver"] = "com.amazon.redshift.jdbc42.Driver"
//})
//.Count();
//.Format("jdbc")
.Format("com.databricks.spark.redshift")
//.Format("com.amazon.redshift.jdbc42.Driver")
.Option("temporary_aws_access_key_id",awsAccessKey)
.Option("temporary_aws_secret_access_key",awsSecretKey)
.Option("url",url)
.Option("dbtable","product_track")
.Option("aws_iam_role",iam_role)
.Option("tempdir",s3Bucket)
.Option("spark.hadoop.fs.s3.access.key",awsAccessKey)
.Option("spark.hadoop.fs.s3.awsSecretAccessKey",awsSecretKey)
.Option("jdbcdriver","com.amazon.redshift.jdbc42.Driver")
.Load();
productState.Show(10);
请注意，我已经证明了许多配置：
没有办法让我工作。例外总是一样：
java.io.IOException: No FileSystem for scheme: s3
at org.apache.hadoop.fs.FileSystem.getFileSystemClass(FileSystem.java:2660)
at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2667)
at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:94)
at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2703)
at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2685)
at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:373)
at com.databricks.spark.redshift.Utils$.assertthatFileSystemIsNotS3BlockFileSystem(Utils.scala:156)
at com.databricks.spark.redshift.RedshiftRelation.<init>(RedshiftRelation.scala:52)
at com.databricks.spark.redshift.DefaultSource.createRelation(DefaultSource.scala:49)
at org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:318)
at org.apache.spark.sql.DataFrameReader.loadV1Source(DataFrameReader.scala:223)
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:211)
at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:167)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.spark.api.dotnet.DotnetBackendHandler.handleMethodCall(DotnetBackendHandler.scala:145)
at org.apache.spark.api.dotnet.DotnetBackendHandler.handleBackendRequest(DotnetBackendHandler.scala:85)
at org.apache.spark.api.dotnet.DotnetBackendHandler.channelRead0(DotnetBackendHandler.scala:28)
at org.apache.spark.api.dotnet.DotnetBackendHandler.channelRead0(DotnetBackendHandler.scala:23)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
at io.netty.handler.codec.MessagetoMessageDecoder.channelRead(MessagetoMessageDecoder.java:102)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
at io.netty.handler.codec.BytetoMessageDecoder.fireChannelRead(BytetoMessageDecoder.java:310)
at io.netty.handler.codec.BytetoMessageDecoder.channelRead(BytetoMessageDecoder.java:284)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1359)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:935)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:138)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:645)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:580)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:497)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:459)
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858)
at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:138)
at java.lang.Thread.run(Thread.java:748)
拜托，hhhhheeeeeeeellllllllppppppppp !!!!
版权声明：本文内容由互联网用户自发贡献，该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容， 请发送邮件至 dio@foxmail.com 举报，一经查实，本站将立刻删除。
