YahooFinanceScheduler.java

package com.tdmconsult.ete.scheduler;

import com.google.common.util.concurrent.RateLimiter;
import com.tdmconsult.ete.masterdata.Security;
import com.tdmconsult.ete.masterdata.SecurityRepository;
import com.tdmconsult.ete.masterdata.SecurityYahooFinanceAddon;
import com.tdmconsult.ete.masterdata.SecurityYahooFinanceAddonRepository;
import com.tdmconsult.ete.services.yahoo.YahooFinanceService;
import com.tdmconsult.ete.services.yahoo.dto.CalendarEvents;
import com.tdmconsult.ete.services.yahoo.dto.SummaryDetail;
import java.util.Objects;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Profile;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.transaction.support.TransactionTemplate;

@Service
@Slf4j
@RequiredArgsConstructor
public class YahooFinanceScheduler {

    private final SecurityRepository securityRepository;
    private final YahooFinanceService yahooFinanceService;
    private final TransactionTemplate transactionTemplate;
    private final SecurityYahooFinanceAddonRepository securityYahooFinanceAddonRepository;

    @Scheduled(cron = "${tdm.yahoo-finance.night-batch.cron}", zone = "Europe/Berlin")
    public void nightBatch() {

        // query maximal e.g. 2 requests per second
        final var rateLimiter = RateLimiter.create(2.0);

        log.info("STARTING yahoo-finance 11night batch");
        final var securities = Objects.requireNonNull(transactionTemplate.execute(status ->
                securityRepository.findAll().stream()
                    .filter(security -> StringUtils.isNotBlank(security.getYahooSymbol()))
                    .toList()), "list of securities cannot be null");
        log.info("There are {} securities identified to participate in this batch", securities.size());

        securities.forEach(security -> {
            try {
                rateLimiter.acquire();
                log.info("now processing {}: {}", security.getIsin(), security.getName());
                transactionTemplate.executeWithoutResult(status -> process(security));
            } catch (final Exception e) {
                log.error("cannot query quote summery for {}", security.getYahooSymbol(), e);
            }
        });
    }

    private void process(final Security security) {

        final var earnings = yahooFinanceService.queryQuoteSummary(security.getYahooSymbol());
        if (earnings == null) {
            log.error("unable to query QuoteSummary from Yahoo Finance - skip");
            return;
        }
        final var nextEarningDate = earnings.getCalendarEvents().flatMap(CalendarEvents::getFirstEarningsDate);
        final var isEarningsDateEstimate = earnings.getCalendarEvents()
                .map(CalendarEvents::getEarnings)
                .map(CalendarEvents.Earnings::getIsEarningsDateEstimate);
        final var previousClose = earnings.getSummaryDetail().flatMap(SummaryDetail::getPreviousClose);
        final var currency = earnings.getSummaryDetail().map(SummaryDetail::getCurrency);

        securityYahooFinanceAddonRepository.findBySecurityIsin(security.getIsin()).ifPresentOrElse(existing -> {
            nextEarningDate.ifPresent(existing::setNextEarningsDate);
            previousClose.ifPresent(existing::setRate);
            currency.ifPresent(existing::setCurrencyIso);
            isEarningsDateEstimate.ifPresent(existing::setIsEarningsDateEstimate);
        }, () -> {
            final var newEntity = SecurityYahooFinanceAddon.of(security);
            nextEarningDate.ifPresent(newEntity::setNextEarningsDate);
            previousClose.ifPresent(newEntity::setRate);
            currency.ifPresent(newEntity::setCurrencyIso);
            isEarningsDateEstimate.ifPresent(newEntity::setIsEarningsDateEstimate);
            securityYahooFinanceAddonRepository.save(newEntity);
        });
    }

    @Configuration
    @RequiredArgsConstructor
    @Profile("XXXlocal")
    public static class NightBatchRunnerOnStartup implements InitializingBean {

        private final YahooFinanceScheduler yahooFinanceScheduler;

        @Override
        public void afterPropertiesSet() throws Exception {
            System.out.println("##### RUNNING NIGHT BATCH ON STARTUP #####");
            yahooFinanceScheduler.nightBatch();
        }
    }
}