如何解决火花提交pyspark脚本超出了最大递归深度
我能够通过spark提交yarn-cluster模式提交org.apache.spark.examples.SparkPi示例jar,并且成功,但是pyspark中的以下代码段失败,最大递归深度超过错误。
package com.example.pricetag.fragments
import android.app.AlertDialog
import android.content.Context
import android.content.DialogInterface
import android.os.Bundle
import android.view.LayoutInflater
import android.view.View
import android.view.ViewGroup
import android.widget.EditText
import android.widget.ImageButton
import android.widget.TextView
import androidx.core.content.ContextCompat
import androidx.fragment.app.Fragment
import androidx.recyclerview.widget.DividerItemDecoration
import androidx.recyclerview.widget.LinearLayoutManager
import androidx.recyclerview.widget.RecyclerView
import com.example.pricetag.R
import com.example.pricetag.fragments.DTO.ItemList
import com.google.android.material.floatingactionbutton.FloatingActionButton
import kotlinx.android.synthetic.main.fragment_shopping_list.*
lateinit var dbHandler: DBHandler
class shopping_list() : Fragment() {
override fun onCreateView(
inflater: LayoutInflater,container: ViewGroup?,savedInstanceState: Bundle?
): View? {
val view = inflater.inflate(R.layout.fragment_shopping_list,container,false)
dbHandler = DBHandler(view.context)
val addItem_button = view.findViewById<FloatingActionButton>(R.id.fab_shoppiglist)
addItem_button.setOnClickListener {
val dialog = AlertDialog.Builder(getContext())
val tempView = layoutInflater.inflate(R.layout.dialog_shoppinglist,null)
val text_result = tempView.findViewById<EditText>(R.id.et_Itemlist)
dialog.setView(tempView)
dialog.setPositiveButton("Add") { _: DialogInterface,_: Int ->
if (text_result.text.isNotEmpty()) {
val itemList = ItemList()
itemList.name = text_result.text.toString()
dbHandler.addItem(itemList)
refreshList()
}
}
dialog.setNeutralButton("DeleteList") { _: DialogInterface,_: Int ->
dbHandler.delete()
refreshList()
}
dialog.setNegativeButton("Cancel"){ _: DialogInterface,_: Int->
}
dialog.show()
}
return view
}
override fun onActivityCreated(savedInstanceState: Bundle?) {
super.onActivityCreated(savedInstanceState)
rv_shoppinglist.layoutManager = LinearLayoutManager(activity)
val verticalDecoration = DividerItemDecoration(
rv_shoppinglist.getContext(),DividerItemDecoration.VERTICAL
)
val verticalDivider =
ContextCompat.getDrawable(activity!!,R.drawable.rv_divider)
verticalDecoration.setDrawable(verticalDivider!!)
rv_shoppinglist.addItemDecoration(verticalDecoration)
}
override fun onResume() {
refreshList()
super.onResume()
}
private fun refreshList(){
rv_shoppinglist.adapter = ShoppinglistAdpter(this.requireContext(),dbHandler,dbHandler.getItem())
}
class ShoppinglistAdpter(val context: Context,dbHandler: DBHandler,val list: MutableList<ItemList>): RecyclerView.Adapter<ShoppinglistAdpter.ViewHolder>(){
override fun onCreateViewHolder(p0: ViewGroup,pl: Int): ViewHolder {
return ViewHolder(LayoutInflater.from(context).inflate(R.layout.rv_child_shoppinglist,p0,false))
}
override fun getItemCount(): Int {
return list.size
}
override fun onBindViewHolder(holder: ViewHolder,p1: Int) {
holder.itemName.text = list[p1].name
holder.itemAmount.text = list[p1].amount.toString()
holder.addButton.setOnClickListener{
dbHandler.changeAmount(list[p1].id.toInt())
}
}
class ViewHolder(v : View) : RecyclerView.ViewHolder(v){
val itemName : TextView = v.findViewById(R.id.tv_item_name)
val itemAmount : TextView = v.findViewById(R.id.tv_item_amount)
val addButton : ImageButton = v.findViewById(R.id.IB_add)
}
}
}
我根据Pyspark on yarn-cluster mode的建议添加了pyspark_python env。
test.py
spark-submit --master yarn --deploy-mode cluster --executor-memory 1G --num-executors 4 --conf spark.yarn.appMasterEnv.PYSPARK_PYTHON="/usr/bin/python2.7" test.py --verbose
如何解决此问题?
from pyspark import SparkContext
from pyspark.sql import HiveContext
sc_new = SparkContext()
SQLContext = HiveContext(sc_new)
SQLContext.setConf("spark.sql.hive.convertMetastoreOrc","false")
txt = SQLContext.sql( "SELECT 1")
txt.show(2000000,False)
- 运行Spark版本1.6.0
- 配置单元1.1.0版
- Hadoop版本:2.6.0-cdh5.13.0
解决方法
通过调用txt.show(2000000,False)
来进行py4j
来进行to-and-fro
jvm-python-object-jvm调用,其中结果没有那么多行。
我相信您最多可以拨入show()
的费用是2000欧元。
为什么您只需要SELECT 1
时为什么需要显示 2000000 条记录?
版权声明:本文内容由互联网用户自发贡献,该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务,不拥有所有权,不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容, 请发送邮件至 dio@foxmail.com 举报,一经查实,本站将立刻删除。