0
votes

I am downloading file(s) from S3 bucket and then trying to read it using MultiResourceItemReader as there could be more then one csv file in the bucket.

I am able to download the files and place it on my local but I am not able to read the files, the files are not in the classpath. I am passing full filepath to read the files, but something is not working as expected. I am getting : No resources to read. Set strict=true if this should be an error condition. I may not be passing the file path correctly.

Custom Log :

2021-04-21 00:32:09.690  INFO 106084 --- [  restartedMain] c.c.S.I.TaskletS3DownloadFiles           : Started Downloading files from S3 bucket..
2021-04-21 00:32:10.086  INFO 106084 --- [  restartedMain] c.c.S.Config.S3BucketConfig              : The input file :  MI4275/input/MI4275sample.csv, has been successfully copied to : C:\Users\Desktop\data\in\MI4275sample.csv, and the file size is : 239505 bytes
2021-04-21 00:32:11.369  INFO 106084 --- [  restartedMain] o.s.batch.core.step.AbstractStep         : Step: [step1MI4275S3ListCopyFiles] executed in 4s697ms
2021-04-21 00:32:16.475  INFO 106084 --- [  restartedMain] o.s.batch.core.job.SimpleStepHandler     : Executing step: [step1ReadLoadMI4275CSV]
2021-04-21 00:32:17.477  WARN 106084 --- [  restartedMain] o.s.b.item.file.MultiResourceItemReader  : No resources to read. Set strict=true if this should be an error condition.
2021-04-21 00:32:20.686  INFO 106084 --- [  restartedMain] o.s.batch.core.step.AbstractStep         : Step: [step1ReadLoadMI4275CSV] executed in 4s210ms
2021-04-21 00:32:23.960  INFO 106084 --- [  restartedMain] o.s.b.c.l.support.SimpleJobLauncher      : Job: [SimpleJob: [name=jobCSProvMI4275]] completed with the following parameters: [{JobID=1618990319394}] and the following status: [COMPLETED] in 20s416ms

Code :

@Configuration
@EnableBatchProcessing
public class JobStepBuilderConfig {
    @Autowired
    JobBuilderFactory jobBuilderFactory;
    @Autowired
    StepBuilderFactory stepBuilderFactory;
    @Autowired
     DataSource datasource;

    @Autowired
    TaskletS3DownloadFiles taskletS3DownloadFiles;

    @Value("${local.input.file.path.pattern}")
    private Resource[] resources;

    String currentDate ;

    @Bean
    @StepScope
    // MultiResourceItemReader to read multiple files sequentially
    public MultiResourceItemReader<Provider> providerMultiResourceItemReader() {
        MultiResourceItemReader<Provider> multiResourceItemReader = new MultiResourceItemReader<>();
        multiResourceItemReader.setResources(resources);
        multiResourceItemReader.setDelegate(providerItemReader());
        return multiResourceItemReader;

    }

    @Bean
    @StepScope
    // FlatfileItemReader to define file properties
    public FlatFileItemReader<Provider> providerItemReader(){
        // create FlatFileItemReader
        FlatFileItemReader<Provider> reader = new FlatFileItemReader<>();
        // skip header
        reader.setLinesToSkip(1);
        DefaultLineMapper<Provider> customerLineMapper = new DefaultLineMapper<>();
        // tokenizer for delimited file
        DelimitedLineTokenizer tokenizer = new DelimitedLineTokenizer();
        customerLineMapper.setLineTokenizer(tokenizer);
        customerLineMapper.setFieldSetMapper(new ProviderFieldSetMapper());
        customerLineMapper.afterPropertiesSet();
        reader.setLineMapper(customerLineMapper);
        return reader ;
    }

    @Bean
    // JdbcBatchItemWriter to write records into database
    public JdbcBatchItemWriter<Provider> providerJdbcBatchItemWriter(){
        JdbcBatchItemWriter<Provider> jdbcBatchItemWriter = new JdbcBatchItemWriter<>();

        jdbcBatchItemWriter.setDataSource(this.datasource);
        jdbcBatchItemWriter.setSql(insertQuery);
        jdbcBatchItemWriter.setItemSqlParameterSourceProvider(new BeanPropertyItemSqlParameterSourceProvider<>());
        jdbcBatchItemWriter.afterPropertiesSet();

        return jdbcBatchItemWriter;

    }



    // Steps
    @Bean
    public Step step1(){
        // step 1 :  Read records from custom table and call stored procedure to update facets table
        return stepBuilderFactory.get("step1S3ListCopyFiles")
                .tasklet(taskletS3DownloadFiles)
                .build();
    }

   @Bean
    public Step step2(){
        // step 2 : Read csv files and dump it into a custom table
        return stepBuilderFactory.get("step2ReadLoadCSV")
                .<Provider, Provider>chunk(1000)
                .reader(providerMultiResourceItemReader())
                .writer(providerJdbcBatchItemWriter())
                .build();
    }


    //  Job
    @Bean
    public Job job(){
        return jobBuilderFactory.get("jobCSProvMI")
                .start(step1())
               .next(step2())
                .incrementer(new RunIdIncrementer())
                .build();
    }
}

application.properties

local.input.file.path.pattern =file:C:/Users/Desktop/data/in/*.csv
1

1 Answers

0
votes

You are declaring a bean definition method (the method providerMultiResourceItemReader() annotated with @Bean) in a class annotated with @Component. This is not correct. You should be declaring your beans in a class annotated with @Configuration instead.

I see no need for your @Component public class FileItemReader { .. } class, you could define the reader in the same class where you define your step, something like:

import org.springframework.batch.core.Job;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.ItemWriter;
import org.springframework.batch.item.support.ListItemReader;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

import java.util.Arrays;

@Configuration
@EnableBatchProcessing
public class MyJobConfiguration {

    @Bean
    public ItemReader<Integer> itemReader() {
        return new ListItemReader<>(Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
    }

    @Bean
    public ItemWriter<Integer> itemWriter() {
        return items -> items.forEach(System.out::println);
    }

    @Bean
    public Job job(JobBuilderFactory jobs, StepBuilderFactory steps) {
        return jobs.get("job")
                .start(steps.get("step")
                        .<Integer, Integer>chunk(5)
                        .reader(itemReader())
                        .writer(itemWriter())
                        .build())
                .build();
    }

}

If you really want to extract the reader's configuration in a separate class, then you need to annotate it with @Configuration:

import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.support.ListItemReader;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

import java.util.Arrays;

@Configuration
public class ItemReaderConfiguration {

    @Bean
    public ItemReader<Integer> itemReader() {
        return new ListItemReader<>(Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
    }
    
}

Then import that class in your main job configuration:

import org.springframework.batch.core.Job;
import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
import org.springframework.batch.core.configuration.annotation.JobBuilderFactory;
import org.springframework.batch.core.configuration.annotation.StepBuilderFactory;
import org.springframework.batch.item.ItemReader;
import org.springframework.batch.item.ItemWriter;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;

@Configuration
@Import(ItemReaderConfiguration.class)
@EnableBatchProcessing
public class MyJobConfiguration {

    @Bean
    public ItemWriter<Integer> itemWriter() {
        return items -> items.forEach(System.out::println);
    }

    @Bean
    public Job job(JobBuilderFactory jobs, StepBuilderFactory steps, ItemReader<Integer> itemReader) {
        return jobs.get("job")
                .start(steps.get("step")
                        .<Integer, Integer>chunk(5)
                        .reader(itemReader)
                        .writer(itemWriter())
                        .build())
                .build();
    }

}