I'm using the Cloudera VM with Centos set up with a single cluster of Hadoop. It uses Eclipse Luna.
I have an UDF written for use with Pig . It's the first time I've written an UDF for Pig. Previous Pig scripts run fine without UDF's. When I run this pig script, I get the following error:
Failed to generate logical plan. Nested exception: org.apache.pig.backend.executionengine.ExecException: ERROR 1070: Could not resolve EasyDates.EasyDateMethods.exec using imports: [, java.lang., org.apache.pig.builtin., org.apache.pig.impl.builtin.]
This error happens in the pig script starting with "CALC_UR_DAYS_BETWEEN". See below.
I've spent 3-4 hours searching on the Internet (and testing) and they all refer to - setting the Classpath properly, - making sure you register your UDF, - making sure the jar file name is the same as the package name, - making sure that the package name is a directory in the working path and is the same name as a the package.
I've done all that yet I still get the error.
As far as I can tell, everything is named properly and where they should be:
- Java package name: EasyDates
- Jar name: EasyDates.jar
- Jar path: /home/cloudera/data/EasyDates/
- Class name: EasyDateMethods
- Set in .bash_profile: CLASSPATH=$CLASSPATH:/usr/jars/:/home/cloudera/data/EasyDates/
I have exhausted the posts after several hours. I cannot find anything else to try. Any other insight is much appreciated!
Java source:
package EasyDates;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import org.apache.pig.EvalFunc;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.util.WrappedIOException;
public class EasyDateMethods extends EvalFunc <String> {
public String exec(Tuple input) throws IOException {
if (input == null || input.size() == 0)
return "0";
try{
Date date1;
Date date2;
String strDiff="0";
int intDiff = 0;
//Get the two string dates from the tuple:
String strDate1 = (String)input.get(0);
String strDate2 = (String)input.get(1);
//Convert them to Dates
date1 = stringToDate(strDate1);
date2 = stringToDate(strDate2);
//The the date difference:
intDiff = getDaysBetween(date1, date2);
//Since I must return the same data type as I call for this Pig method, this converts the
//difference in days to a string.
return Integer.toString(intDiff);
}catch(Exception e){
throw WrappedIOException.wrap("Caught exception processing input row ", e);
}
}
private Date stringToDate(String theDateString) {
//Make sure the Pig script formats the date format this way or whatever format you choose.
//Just make sure they agree.
SimpleDateFormat dateFormatter = new SimpleDateFormat ( "dd-MMM-yyyy" );
String dateInString = "12-May-2014";
Date theDate;
java.util.Date dateObject = null;
try {
dateObject = dateFormatter.parse ( theDateString );
System.out.println( dateObject );
System.out.println( dateFormatter.format ( dateObject ) );
//theDate = dateFormatter.format ( dateObject );
} catch ( Exception e) {
System.out.println( e.getMessage() + " " + e.getStackTrace() );
};
return dateObject ;
}
static int getDaysBetween(Date curDate, Date prevDate) {
//Precondition: the difference in days between the current meter read date and the last one is not known
//Postcondition: the difference in days between the current meter read date and the last one is known
Calendar currentDate = Calendar.getInstance();
Calendar previousDate = Calendar.getInstance();
currentDate.setTime(curDate);
previousDate.setTime(prevDate);
int theDiffinDays = 0;
int theDiffinYears = 0;
int currentDay;
int previousDay;
int currentYear;
int previousYear;
try {
currentDay = currentDate.get(Calendar.DAY_OF_YEAR);
System.out.println("currentDay is " + currentDay);
previousDay = previousDate.get(Calendar.DAY_OF_YEAR);
System.out.println("previousDay is " + previousDay);
currentYear = currentDate.get(Calendar.YEAR);
System.out.println("currentYear is " + currentYear);
previousYear = previousDate.get(Calendar.YEAR);
System.out.println("previousYear is " + previousYear);
if (currentYear == previousYear) {
theDiffinDays = currentDay - previousDay;
}
else
{
theDiffinYears = currentYear - previousYear;
//This assumes 2 contiguous years, eg 2016 and 2017; so this wouldn't work if the diff in years is greater than 1
if (isLeapYear(previousYear)) {
//The following has not been corrected for leap year:
//If the previous year is a leap year
theDiffinDays = 366 - previousDay + currentDay;
}
else {
//If the current year is a leap year or neither year is a leap year: (because the day of year should be inherent whether leap or not)
theDiffinDays = 365 - previousDay + currentDay;
}
}
//return theDiffinDays;
}
catch (Exception ex){
System.out.println(ex.getMessage() + " " + ex.getStackTrace());
}
return theDiffinDays;
}
private static boolean isLeapYear(int theYear){
//Precondition: the year is not designated as a leap year or not
boolean ans = false;
try {
switch (theYear){
case 2004: ans = true;
break;
case 2008: ans = true;
break;
case 2012: ans = true;
break;
case 2016: ans = true;
break;
case 2020: ans = true;
break;
case 2024: ans = true;
break;
case 2028: ans = true;
break;
case 2032: ans = true;
break;
case 2036: ans = true;
break;
case 2040: ans = true;
break;
case 2044: ans = true;
break;
case 2048: ans = true;
break;
default: ans = false;
}
}
catch (Exception ex){
System.out.println(ex.getMessage() + " " + ex.getStackTrace());
}
return ans;
}
}
Pig Script:
--Simple Pig script to read in a file with dates, and pass the dates to the EasyDate class
REGISTER /home/cloudera/data/EasyDates/EasyDates.jar;
DEFINE DaysBetween EasyDates.EasyDateMethods;
----------------------------------------------------Load the file--------------------------------------------
--The file needs two different dates in one row for this test
devicePageCountAll = LOAD 'Data_For_Test_Jar.txt' USING PigStorage('\t')
AS (
account_code:chararray,
serial_number:chararray,
reported_date:chararray,
reported_date2:chararray);
--dump devicePageCountAll;
--------------------------------------------------Get the date difference in days and store the result-----------------
devicePageCountAll2 = foreach devicePageCountAll {
CALC_UR_DAYS_BETWEEN = DaysBetween((ToString(REPLACE(reported_date, '\\"', ''), 'yyyy-MM-dd')), (ToString(REPLACE(reported_date2, '\\"', ''), 'yyyy-MM-dd')));
generate
account_code,
serial_number,
reported_date,
reported_date2,
(CALC_UR_DAYS_BETWEEN > 15000 ? 0 : CALC_UR_DAYS_BETWEEN) AS days_since_last_reported;
}
dump devicePageCountAll2;
Thanks!
-Dpig.additional.jars='EasyDates.jar'instead of register incase you dont want to specify register at the top of the pig file. – rahulbmv