0
votes

Can anyone tell me why is this returning a Missing parameter type error please? it is pointing on iterator inside the foreach instruction. The aim is to insert DF inside an Oracle table. Thank you:

def insertTechRejet(df: DataFrame): Boolean = {
    var committed = false
    var ik: Long = 99999999
    var iterator: Iterator[Row] = null
    val connexion = SparkCP_mod.pool().getConnection
    var modificationStmt = ""
    modificationStmt = """INSERT INTO DWG_TECH_REJET 
                (ID_TECH_REJET, NOM_TABLE, TYPE_TRAITEMENT, DT_REJET, CD_REJET, CD_SEVERITE_REJET, TOP_A_RECYCLER, TOP_REJET_TRAITE, DT_REJET_TRAITE, NOM_TABLE_ORIG, CAUSE_REJET)
                VALUES(?,?,?,?,?,?,?,?,?,?,?)
                """
      var retour: Boolean = false
      val stmt = connexion.prepareStatement(modificationStmt)
      connexion.setAutoCommit(true)
      df.foreach(iterator =>
        try {

        ik=ik+1
        stmt.setLong(1, ik)
        stmt.setString(2, iterator(1))
        stmt.setString(3, iterator(2))
        stmt.setTimestamp(4, iterator(3))
        stmt.setString(5, iterator(4))
        stmt.setString(6, iterator(5))
        stmt.setInt(7, iterator(6))
        stmt.setInt(8, iterator(7))
        stmt.setTimestamp(9, iterator(8))
        stmt.setString(10, iterator(9))
        stmt.setString(11, iterator(10))


      stmt.executeQuery()

      true
    } catch { ......

I also tried df.foreach(iterator: Row => and it doesn't work

1

1 Answers

2
votes

I believe you can make use of Spark SQL's JDBC datasource to write a dataframe to an Oracle table.

Details you can get from https://spark.apache.org/docs/latest/sql-data-sources-jdbc.html

All you need to do is to pass proper version of Oracle JDBC jar using --jars when you are submitting your spark job.