从Spark数据框中的其他ROW获取值

如何解决从Spark数据框中的其他ROW获取值

我和下面的DF一起住

+------+------+----+
|  Year|    PY| VAL|
+------+------+----+
|202005|201905|2005|
|202006|201906|2006|
|202007|201907|2007|
|201905|201805|1905|
|201906|201806|1906|
|201907|201807|1907|
|201805|201705|1805|
|201806|201706|1806|
|201807|201707|1807|
+------+------+----+

由

获得

val df1=Seq(
("202005","201905","2005"),("202006","201906","2006"),("202007","201907","2007"),("201905","201805","1905"),("201906","201806","1906"),("201907","201807","1907"),("201805","201705","1805"),("201806","201706","1806"),("201807","201707","1807")
)toDF("Year","PY","VAL")

我想在单独的列中填充上一年的值（VAL_PY）。该值实际上位于同一DF的不同行中。

此外，我想以分布式方式实现这一目标，因为我的DF非常庞大（> 1000万条记录）

预期输出-

+------+------+----+-------+
|  Year|    PY| VAL| VAL_PY|
+------+------+----+-------+
|202005|201905|2005|1905   |
|202006|201906|2006|1906   |
|202007|201907|2007|1907   |
|201905|201805|1905|1805   |
|201906|201806|1906|1806   |
|201907|201807|1907|1807   |
|201805|201705|1805|null   |
|201806|201706|1806|null   |
|201807|201707|1807|null   |
+------+------+----+-------+

解决方法

#if os(iOS)
import UIKit
import DateToolsSwift

open class DayViewController: UIViewController,EventDataSource,DayViewDelegate {
  public lazy var dayView: DayView = DayView()
  public var dataSource: EventDataSource? {
    get {
      return dayView.dataSource
    }
    set(value) {
      dayView.dataSource = value
    }
  }

  public var delegate: DayViewDelegate? {
    get {
      return dayView.delegate
    }
    set(value) {
      dayView.delegate = value
    }
  }

  public var calendar = Calendar.autoupdatingCurrent {
    didSet {
      dayView.calendar = calendar
    }
  }

  open override func loadView() {
    view = dayView
  }

  override open func viewDidLoad() {
    super.viewDidLoad()
    edgesForExtendedLayout = []
    view.tintColor = SystemColors.systemRed
    dataSource = self
    delegate = self
    dayView.reloadData()

    let sizeClass = traitCollection.horizontalSizeClass
    configureDayViewLayoutForHorizontalSizeClass(sizeClass)
  }

  open override func viewDidAppear(_ animated: Bool) {
    super.viewDidAppear(animated)
    dayView.scrollToFirstEventIfNeeded()
  }

  open override func willTransition(to newCollection: UITraitCollection,with coordinator: UIViewControllerTransitionCoordinator) {
    super.willTransition(to: newCollection,with: coordinator)
    configureDayViewLayoutForHorizontalSizeClass(newCollection.horizontalSizeClass)
  }

  open func configureDayViewLayoutForHorizontalSizeClass(_ sizeClass: UIUserInterfaceSizeClass) {
    dayView.transitionToHorizontalSizeClass(sizeClass)
  }
  
  // MARK: - CalendarKit API
  
  open func move(to date: Date) {
    dayView.move(to: date)
  }

  open func reloadData() {
    dayView.reloadData()
  }

  open func updateStyle(_ newStyle: CalendarStyle) {
    dayView.updateStyle(newStyle)
  }

  open func eventsForDate(_ date: Date) -> [EventDescriptor] {
    return [Event]()
  }

  // MARK: - DayViewDelegate

  open func dayViewDidSelectEventView(_ eventView: EventView) {
  }

  open func dayViewDidLongPressEventView(_ eventView: EventView) {
  }

  open func dayView(dayView: DayView,didTapTimelineAt date: Date) {
  }
  
  open func dayViewDidBeginDragging(dayView: DayView) { 
  }

  open func dayView(dayView: DayView,willMoveTo date: Date) {
  }

  open func dayView(dayView: DayView,didMoveTo date: Date) {
  }

  open func dayView(dayView: DayView,didLongPressTimelineAt date: Date) {
  }

  open func dayView(dayView: DayView,didUpdate event: EventDescriptor) {
  }
  
  // MARK: - Editing
  
  open func create(event: EventDescriptor,animated: Bool = false) {
    dayView.create(event: event,animated: animated)
  }

  open func beginEditing(event: EventDescriptor,animated: Bool = false) {
    dayView.beginEditing(event: event,animated: animated)
  }
  
  open func endEventEditing() {
    dayView.endEventEditing()
  }
}
#endif

输出：

DatePickerControllerDelegate

好像是左自我连接。如果我想念什么，请告诉我。