import logging
import pandas as pd
import numpy as np
from os import path
from byneuron import Entity
from datetime import datetime,  timedelta
from .configurator import CEDAggregation, CEDTarget, CEDSource, CHEventChange, CHMonotonic
from .handler import CalculationHandlerEntity
from pandas.api.types import is_numeric_dtype

from .configurator import CustomJob

log = logging.getLogger('calc/calc')

class CalculationEntity():#CalculationHandlerEntity):

    def __init__(self, che:CalculationHandlerEntity):
        self.che = che
        #super(CalculationEntity, self).__init__(entity=che.entity, handler=che.bn, log_series=che.log_series)
        # self._query()

    def __repr__(self):
        return f'CalulationEntity for {self.che.entity}'

    @staticmethod
    def fix_numeric(df:pd.Series, label:str):
        if not is_numeric_dtype(df):
            log.warning(f'dataframe for {label} in unexpected format: %s\n%s', df.dtype, df.head())
            fix = pd.to_numeric(df, errors='coerce')
            error = df[~fix.notna()]
            if not error.empty:
                log.info('cause of wrong format %s', error)
            df = fix
        return df

    def calculate_targets(self):
        """simple handle to excecute all jobs """
        return self.che.handle(handle_function=self.handle_targets)
        # jobs=None is the default setting; running all jobs

    def calculate_default_targets(self):
        """simple handle to excecute the default job """
        return self.che.handle(handle_function=self.handle_targets, jobs=[CustomJob()])

    def fix_targets(self):
        return self.che.handle(handle_function=self._duplicate_events)

    def handle_targets(self, job:CustomJob, from_date: datetime, to_date: datetime) -> None:
        """
        procedure to handle a task:
        0/ validate job
        1/ retrieve and merge dataSeries for each source
        2/ perform calculation
        3/ store data
        :param job: Job (verified with job in CEDTarget, e.g. Job.DEFAULT)
        :param from_date: earliest time (inclusive)
        :param to_date: (exclusive)
        :return:
        """
        # note: this function does not buffer any data;
        #       other instances could be processing data too

        # iterate targets
        log.info('handle_targets with %s', job)
        sources_by_label = {s.label:s for s in self.che.sources}
        aggr_df = {}
        # we will not do this assync , this will complicate buffer source data
        for t in self.che.targets:
            insights = []
            self.che.counter.time_start()
            # continue if job not match target
            if t.job != job:
                continue
            if self.che.handler_use_cron_jobs:
                if not t.match_cron():
                    log.debug('cron blocked job %s on target %s', job, t)
                    continue
            if not t.valid_calculation:
                # avoid retrieving dataSeries when the calculation would fail
                continue
            if not all([l in sources_by_label for l in t.source_labels]):
                # e.g. typo in calculation
                log.warning('missing sources for target %s: %s', t, [l for l in t.source_labels if l not in sources_by_label])
                continue
            df_calculate = []
            for s_label in t.source_labels:
                df_label = f'{t.interval.value};{s_label}'
                if not df_label in aggr_df:
                    dfs = []
                    source: CEDSource = sources_by_label.get(s_label)
                    log.info(f'retrieve data from source {source}')
                    for e, r_from_date, r_to_date in source.iter_entities():
                        df = self.get_aggregation_df(
                            source = source,
                            entity = e,
                            target = t,
                            from_date = max(from_date, r_from_date),
                            to_date = min(to_date, r_to_date)
                        )
                        #assure numeric dtype
                        dfs.append(self.fix_numeric(df, f'source {source.label} {e.public_id}'))
                    for i, (number_value, r_from_date, r_to_date) in enumerate(source.iter_meta_values()):
                        start = max(from_date, r_from_date)
                        end = min(to_date, r_to_date)
                        freq = t.interval.pd_interval
                        dates = pd.date_range(start=start,end=end, freq=freq)
                        df = pd.Series(number_value, index=dates).rename(f'{source.label}_{i}')
                        dfs.append(self.fix_numeric(df, f'source {source.label} {i}'))
                    insights.extend(dfs)
                    df = source.merge(dfs).rename(s_label)

                    log.debug('Source retrieved %s, %s', df_label, df)
                    aggr_df.update({df_label: df})
                df = aggr_df.get(df_label)
                df_calculate.append(df)
                insights.append(df)
            # make calculation
            df = t.calculation(df_calculate)
            insights.append(df)
            # store value - rounding on target is done at store phase
            self.store(df, t, from_date, to_date)
            self.che.counter.time_stop(t.name)
            if self.che.handler_ch_export_xls and insights:
                df_export = pd.concat(insights,axis=1)
                filename = f"insight_{self.che.entity.indexset_name}_{self.che.entity.public_id}.xlsx"
                self.save_df_to_excel(df_export, filename, t.name)
                log.info('export to excell \n%s', df_export.head().to_string())

    def save_df_to_excel(self, df, f, s):
        """
        Save df (numpy array or DataFrame) to Excel file f, sheet s.

        - Converts numpy array to DataFrame using default integer index.
        - Creates file if not exists.
        - Creates sheet if not exists.
        - Upserts: replaces row if index exists, appends if not.
        - Sorts final sheet index descending (large to small).

        Handles tz-aware datetime indexes by making naive for Excel.
        """

        def normalize_tz(index):
            """Safely normalize any datetime index to UTC-naive for Excel."""
            if index.dtype == 'datetime64[ns]':  # Naive
                # Assume UTC and keep naive (no tz_convert needed)
                pass
            elif index.dtype == 'datetime64[ns, UTC]':  # Aware
                # Convert to UTC then strip timezone
                index = index.tz_localize(None)
            elif hasattr(index, 'tz'):  # Aware
                # Convert to UTC then strip timezone
                index = index.tz_convert('UTC').tz_localize(None)
            elif index.dtype == 'object':
                index = pd.to_datetime(index, utc=True).tz_convert('UTC').tz_localize(None)
            return index

        # Handle numpy array input
        if isinstance(df, np.ndarray):
            df = pd.DataFrame(df)
        # Copy to avoid modifying original
        df_work = df.copy()

        # Handle tz-aware index (common with your datetime data)
        df_work.index = normalize_tz(df_work.index)
        if not path.exists(f):
            # Case 1: Create new file and save
            df_work.to_excel(f, sheet_name=s, index=True)
            return 'Created new file and sheet.'

        # File exists
        try:
            existing = pd.read_excel(f, sheet_name=s, index_col=0)
            # Handle tz-aware index (common with your datetime data)
            existing.index = normalize_tz(existing.index)
        except ValueError:
            # Case 2: Sheet missing, append it
            with pd.ExcelWriter(f, engine='openpyxl', mode='a') as writer:
                df_work.to_excel(writer, sheet_name=s, index=True)
            return 'Created new sheet in existing file.'

        # Case 3: Sheet exists - upsert
        # Append new indices only
        new_rows = df_work[~df_work.index.isin(existing.index)]
        if not new_rows.empty:
            if existing.empty:
                existing = new_rows
            existing = pd.concat([existing, new_rows])
        # Replace/update matching indices
        existing.update(df_work)
        # Sort index descending (large to small)
        existing_sorted = existing.sort_index(ascending=False)

        # Save back, replacing sheet
        with pd.ExcelWriter(f, engine='openpyxl', mode='a', if_sheet_exists='replace') as writer:
            existing_sorted.to_excel(writer, sheet_name=s, index=True)
        return 'Updated existing sheet with upsert and descending sort.'


    def get_aggregation_df(self, source: CEDSource, entity: Entity, target: CEDTarget,
                           from_date: datetime, to_date: datetime) -> pd.Series:
        default = pd.Series(dtype=float).rename(entity.public_id)
        log.info(f'get_aggregation {source.aggregation} for {entity} with range ({from_date} > {to_date})')
        if not (isinstance(source, CEDSource) and isinstance(entity, Entity) and isinstance(target, CEDTarget) and
                isinstance(from_date, datetime) and isinstance(to_date, datetime)):
            return default

        interval = target.interval
        aggregation = source.aggregation
        rounding = source.rounding
        # rounddown to have only full interval periods
        # from_date = interval.round_down(from_date)
        # to_date = interval.round_down(to_date)

        if from_date >= to_date:
            log.warning('from_date/to_date error %s, %s', from_date, to_date)
            return default
        shifted_from_date = from_date
        if aggregation.add_rows:
            # extended first row
            shifted_from_date = interval.shift_datetime(from_date, aggregation.add_rows)
        records = [i for i in self.che.aggregation_records(entity, aggregation, interval, shifted_from_date, to_date)]
        log.info('get_aggregation with %s records', len(records))
        df = pd.DataFrame.from_records(records)
        log.debug('get_aggregation with df \n%s', df.head().to_string())
        # if no data is available, their will be a column count (all values are 0), but no other aggrs
        required = ['time','count'] + aggregation.aggrs
        if not self.is_dataframe(df, required):
            # no data
            return default
        df.set_index('time', drop=True, inplace=True)
        column_scaled = df.columns.difference(['count','cardinality']) # avoid getting scaled on column 'count'
        df[column_scaled] = df[column_scaled].mul(self.che.scalar(entity)).round(rounding)
        if aggregation == CEDAggregation.MONOTONIC:
            log.debug('get_aggregation MONOTONIC from %s', df.head().to_string())
            df.sort_index()  # ascending
            df['diff'] = df['close'].diff()
            df_extended = df
            #df = df.drop(df.index[0])
            if (entity.meta_booleans or {}).get('monotonic', True) is False:
                # metaBoolean momotonic is explicitly False > processing of non-accumulating counters
                log.info('processing non-accumulating counter %s', entity)
                df = df['sum'].rename(entity.public_id)  # first row is extended
            elif df['diff'].lt(0).any():
                log.debug('not monotonic %s\n%s', entity, df[df['diff'].lt(0)].to_string())
                if self.che.handler_monotonic is CHMonotonic.ACCEPT:
                    df = df['diff']
                elif self.che.handler_monotonic is CHMonotonic.IGNORE:
                    df = df['diff'].mask(df['diff'] < 0, np.nan)
                elif self.che.handler_monotonic is CHMonotonic.BLOCK:
                    df = df['diff'].mask(df['diff'] < 0, False)
                elif self.che.handler_monotonic in [CHMonotonic.FIX, CHMonotonic.FIX_WITH_EVENTS]:
                    df = self.monotonic_fix(df_extended, entity, target)
            else:
                df = df['diff']
        elif aggregation == CEDAggregation.PEAK:
            log.debug('get_aggregation PEAK from %s', df.head().to_string())
            df=df[['max', 'min']].abs().max(axis=1)
        elif aggregation == CEDAggregation.AVG_7D:
            # forecast based on avg of last 7days
            # half_window is 1h
            half_window = interval.intervals_per_hour
            df = df[aggregation.aggr]
            if isinstance(half_window, int):
                df = self.gaussian_smooth(df['avg'],half_window)
            df = self.rolling_avg(df, 7)

        elif aggregation.aggr and aggregation.aggr in df.columns:
            log.debug('get_aggregation %s from %s', aggregation.aggr, df.head().to_string())
            df=df[aggregation.aggr]
        else:
            log.warning('get_aggregation not found for %s', aggregation)
            return default
        if aggregation.on_change:
            df[df == df.shift(1)] = np.nan # True where value equals previous
        if shifted_from_date != from_date:
            df = df.loc[from_date:] # drop extended rows
        return df.round(rounding).rename(entity.public_id)

    def round_interval_dates(self, from_date, to_date, interval):
        # issue; say need data from 8:15 till 9:15 but hourly interval, we need 8:00 till 9:00
        # same for daily or montly
        from_date = interval.round_down(from_date)
        to_date = interval.round_down(to_date)
        return from_date, to_date

    def gaussian_smooth(self, df: pd.Series, half_window: int):
        window = half_window*2  # Fixed number of rows (x left + current + x right with center=True).
        # std value	Effect	Use case
        # std=window	Almost uniform	Minimal smoothing
        # std=1	Strong center weighting	Noise reduction
        # std=window/6	Default heuristic	Balanced smoothing
        df.rolling(window, win_type="gaussian", center=True).mean(std=window/6)
        return df
    def rolling_avg(self, df:pd.Series, days:int):
        df = df.groupby(df.index.time).rolling(
            f'{days}D',
            closed='left',  # Previous days only
            min_periods=1
        ).mean().droplevel(0).reindex(df.index) #droplevel removes multi-index
        return df

    def rolling_gaussian_avg(self, df:pd.Series, days:int):
        # just an idea > would be bad for e.g. weekly patterns,
        # e.g. monday forecast would be influenced more by weekend values
        # for 7days > std=7/3 > weigths: ['0.377', '0.286', '0.181', '0.095', '0.042', '0.015', '0.005']
        df = df.groupby(df.index.time).rolling(
            f'{days}D',
            closed='left',  # Previous days only
            win_type='gaussian',
            min_periods=1
        ).mean(std=days/3).droplevel(0).reindex(df.index) #droplevel removes multi-index
        return df

    def monotonic_fix(self, df:pd.DataFrame, item:Entity, target:CEDTarget):
        """
        uses different strategies to fix a non-monotonic serie

        Conditions for escape
        1/ if not slightly monotonic > block all data
        2/ if no problems noticed > return diff

        Strategies applied:
        1/ check for (partial) resets > diff = (max - open) + (close - min)
        2/ check for inconsistent events using the averages > currently not handled
        3/ check for small drops (rounding, errors, ..) -> set to 0
        4/ remaining cases are handled by analysing the events individually
        """
        # analyse the dataframe
        log.debug('monotonic_fix for df %s', df)
        df.sort_index()  # ascending
        df['avg_diff'] = df['avg'].diff()  # compare average over open
        df['close_diff'] = df['close'].diff()
        # df = df.drop(df.index[0])
        avg_diff_notnull = df['avg_diff'][df['avg_diff'] != 0]
        avg_diff_positive_ratio = round(avg_diff_notnull.gt(0).mean(),3)
        has_drop = ((df['avg_diff'].lt(0)) | (df['close_diff'].lt(0)))
        log.info('starting monotonic_analyse for %s with positive ratio %s\n%s', item, avg_diff_positive_ratio, df[has_drop | has_drop.shift(1) | has_drop.shift(-1)].to_string())
        if avg_diff_positive_ratio < 0.90:
            # idea: at least 85% of the avg.diff is positive => Alert that this device will be blocked
            log.warning('monotonic_analyse > %s not incremental', item)
            df['diff'] = False  # fill with False
            return df['diff']
        elif has_drop.sum() == 0:
            log.info('monotonic_analyse > %s is incremental', item)
            df['diff'] = df['close_diff']
            return df['diff']
            # with the averages being smooth; no structural problems are expected.
            # suggest to process with event-wise diff for these periods
        # use close_diff; but remove the negative drops
        df['diff'] = df['diff'].where(~has_drop, np.nan)
        log.info('monotonic_analyse > %s candidate for fix \n%s', item, df[has_drop].to_string())

        # 1/ check for meter resets:
        # full reset: pct change > threshold (90%)
        df['reset'] = df['close'].pct_change().lt(-0.90)
        # partial reset: negative drop >> positive increase (larger than the known max increase)
        if len(df.index) > 100:
            # avoid the situation where a value is temporarily increased, so drop must be bigger than max increase .
            # limitation when too little data is available to understand size of diff
            max_increase = df['close_diff'].max()
            df['reset'] = (df['reset']) | (df['close_diff'].lt(-2 * max_increase))
        if df['reset'].sum() > 0:
            mask = df['reset'] & df['diff'].isna()
            calc = np.maximum(0, df['max'] - df['min'] + df['close_diff'])
            df['diff'] = df['diff'].mask(mask, calc)
            log.info('monotonic_analyse > fix > reset detected \n%s', df[df['reset']].to_string())

        # 2/ buckets with hidden issues :
        df['hidden'] = (has_drop) & (df['close_diff'].ge(0))
        if df['hidden'].sum() > 0:
            log.info('monotonic_analyse > fix > hidden errors detected \n%s', df[df['hidden']].to_string())
            df['diff'] = df['diff'].where(~df['hidden'], df['close_diff'])

        # 3/ small drops
        if len(df.index) > 100:
            # need enough to have significant step
            average_increment = df[df['close_diff'].ge(0)]['close_diff'].mean()
            threshold_increment = -0.1 * average_increment
            df['small'] = df['close_diff'].between(threshold_increment, 0, inclusive="neither")
            if df['small'].sum() > 0:
                mask = df['small'] & df['diff'].isna()
                df['diff'] = df['diff'].mask(mask, 0)
                log.info('monotonic_analyse > fix > small (<< %s) errors detected \n%s', average_increment,
                          df[df['small']].to_string())

        # 4/ other cases
        if self.che.handler_monotonic == CHMonotonic.FIX_WITH_EVENTS:
            #delta = df.index.mean()
            interval = target.interval
            count = df['count'].max()
            ## find
            not_monotonic = df['diff'].isna()
            work_this_index = list(df[not_monotonic].index)
            while len(work_this_index):
                i = work_this_index.pop(0)
                event_based_diff = self.fix_with_events(
                    entity=item,
                    from_date=interval.shift_datetime(i, 2),
                    to_date=interval.shift_datetime(i, -8),
                    count=max(1000, count * 10),
                    sample=interval.value
                )
                for j in event_based_diff.index:
                    if j == i:
                        pass
                    elif j in work_this_index:
                        work_this_index.remove(j)
                    else:
                        continue
                    diff_events = event_based_diff[j]
                    if diff_events >= 0:
                        log.info('insert diff from events %s at %s', diff_events, j)
                        df.loc[i, 'diff']=diff_events

            log.info('monotonic_analyse FIX_WITH_EVENTS > %s \n%s', item, df[not_monotonic].to_string())
        return df['diff']

    def fix_with_events(self, entity, from_date, to_date, count, sample):

        def is_monotonic(df: pd.DataFrame) -> bool:
            if self.is_dataframe(df, 'value'):
                not_monotonic = df['value'].diff() < 0
                if not_monotonic.any():
                    return False
            return True
            #    return df['value'].is_monotonic_increasing

        def fix_zero(df: pd.DataFrame) -> pd.DataFrame:
            # 0. remove continuous zero values
            if is_monotonic(df):
                return df
            is_zero = (df['value'] == 0.0) & (
                    df['value'].shift(1) == 0.0)  # this event is zero and the previous one too
            mark_events(df[is_zero], 'false_zero')
            return df[~is_zero]

        def fix_sync(df: pd.DataFrame) -> pd.DataFrame:
            if is_monotonic(df):
                return df
            if len(df.index) < 100:
                return df # "Insufficient data"

            delay = (df['created'] - df['datetime']).dt.total_seconds()  # .gt(15 * 60)
            delta_created = df['created'].diff().dt.total_seconds()
            delta_event = df['datetime'].diff().dt.total_seconds()
            large_delay = delay.gt(86400)  # more than 1 day 24*3600seconds
            # cases:
            # 1/ serie with backfilled data: creation dates for events are similar - ignore
            #  / note for cluster senders; these fire a set of different events at the same time (different timeseries) + period of less than 15 minutes
            # 2/ serie with faulty data (node in wrong time) : delta between creation and event are contant and above reasonable threshold \
            # 3/ wierd bug were few events are processed with wrong event time

            # 0/ future events; with creation date before event date > ignore , but do not remove
            future = delay.lt(0)
            if future.any():
                log.info('fix_with_events > remove future events %s', df[future].to_string())
                df = df[~future]
            # 1/ backfilled data
            # is the time between events much larger than time between creation
            # read like: every 15 minutes event, but inserted every hour
            if delta_event.median() > abs(100 * delta_created.median()):
                #avoid mean for outliers due to e.g. period offline
                log.info('fix_with_events > fix_sync skip backfilled data')
                pass
            # 2 faulty time on node
            elif large_delay.any() :
                large_delay_mean = delay[large_delay].mean()
                large_delay_mad = (delay[large_delay] - large_delay_mean).abs().median()  # spread of the large delays
                if large_delay_mad < 3600:
                    # not sure if behaviour of wrong timeservers 'locks' or 'shifts'  time
                    pass
                log.info('fix_with_events > fix_sync has large delay data: mean: %s h, mad: %s h',
                         round(large_delay_mean / 3600, 1),
                         round(large_delay_mad / 3600, 1))
                # examples 133/5; 70/0; 242/55; 141/3
                mark_events(df[large_delay], 'delayed')
                df = df[~large_delay]
            # 3 wrong inserted data: inserted = creation time is before the next events
            created_numeric = (df['created']-df['created'].min()).dt.total_seconds()
            future_creationdates = created_numeric.iloc[::-1].shift(1).rolling('1h').min().iloc[::-1]
            #df['delta_future'] = min_future_creationdates - created_numeric
            created_in_future = created_numeric.gt(future_creationdates)
            if created_in_future.any():
                # in near future, there is an event with an earlier creation date
                mark_events(df[created_in_future], 'desync')
                df = df[~created_in_future]

            return df

        def fix_outliers(df: pd.DataFrame) -> pd.DataFrame:

            if is_monotonic(df):
                return df
            # 2. remove extreme outliers in diff/dt causing nearby discontinuations
            # use IQR with wide fencing since powers form a skewed distribution
            avg_power = df['value'].diff() / df['datetime'].diff().dt.total_seconds()
            marked = (avg_power.lt(0.0) | avg_power.shift(-1).lt(0.0))
            if marked.sum() == 0:
                # duplicate ? ~ no drops should already be detected with is_monotonic
                return df
            # Calculate Q1 (25th percentile) and Q3 (75th percentile)
            q1 = avg_power.quantile(0.25)
            q3 = avg_power.quantile(0.75)
            iqr = q3 - q1
            # Calculate lower and upper bounds for detecting outliers
            fence = 3  # 1.5 is normal outlier; 2 or 3 are used for wider fences
            lower_bound = q1 - fence * iqr
            upper_bound = q3 + fence * iqr
            # Filter out outliers
            outliers = (avg_power < lower_bound) | (avg_power > upper_bound)
            noise = marked & outliers
            mark_events(df[noise], 'outlier')
            return df[~noise]

        def mark_events(df: pd.DataFrame, reason: str) -> None:
            """
            here we fix events that are not compliant
            df: dataframe with at least a column 'id' that has the events to delete
            """
            if not self.is_dataframe(df, ['id']):
                return
            log.info('fix_with_events > drop %s events for %s: \n%s', self.che.entity, reason, df.to_string(max_rows=20))
            event_ids = df['id'].tolist()
            events = [e.set_tag(reason) for e in self.che.event_record_keys(keys=event_ids)]
            self.che.bn.update(events, delete=True)

        records = [
            {
                # 'deleted': event.deleted,
                # 'item': event.item_id,
                'id': e.id,
                'datetime': e.datetime,
                'value': e.value,
                'created': e.datetime_created
            }
            for e in self.che.event_records_dates(entity, from_date, to_date, count+1)]
        df = pd.DataFrame.from_records(records)
        df = df.set_index('datetime', drop=False)
        df = df.sort_index()
        df = fix_zero(df)
        df = fix_sync(df)
        df = fix_outliers(df)
        log.debug('monotonic_events_df > end fix \n %s', df.tail().to_string())
        df['diff'] = df['value'].diff()
        log.debug('monotonic_events_df > end fix \n %s', df.tail().to_string())
        df_diff = df["diff"].resample(sample).agg(pd.Series.sum, skipna=False)
        df_diff = df_diff.iloc[1:]  # remove first row
        return df_diff

    @staticmethod
    def is_dataframe(df, columns=None):
        if isinstance(df, pd.DataFrame) and not df.empty:
            if columns is None:
                return True
            elif isinstance(columns, list):
                return all(c in df.columns for c in columns)
            else:
                return columns in df.columns
        return False

    def store(self, df:pd.Series, target: CEDTarget, from_date:datetime, to_date:datetime):
        if not isinstance(df, pd.Series):
            log.warning('store %s with df %s in target %s', self.che.entity, df, target)
            return
        # the numeric error should be catches at the calculation phase
        # df=self.fix_numeric(df, target.name)
        if not isinstance(target, CEDTarget) or not target.entity:
            log.warning('store %s missing target %s', self.che.entity, target)
            return
        log_tag = target.entity.public_id
        if not isinstance(target.scalar,float) or not target.scalar:
            log.warning('store %s missing scalar in target %s', self.che.entity, target)
            return
        if not isinstance(target.rounding, int):
            log.warning('store %s missing rounding in target %s', self.che.entity, target)
            return
        # set scaling according to unit
        # df = df.replace([np.inf, -np.inf], np.nan).dropna().div(target.scalar).round(target.rounding)
        if df.empty:
            if self.che.handler_event_change == CHEventChange.UPDATE:
                # explicit remove events
                log.warning('store %s purging events in target %s from %s to %s', self.che.entity, target, from_date, to_date)
                events_delete = []
                for event in self.che.event_records_dates(target.entity, from_date, to_date + timedelta(seconds=1),
                                                          2 * df.size):
                    events_delete.append(event)
                self.che.events_delete_with_counter(events_delete, log_tag=log_tag)
            return
        log.info('store %s %s', target, df.head())
        events_update = []
        events_delete = []
        events_create = []
        # check with existing values
        if self.che.handler_event_change != CHEventChange.IGNORE:
            for event in self.che.event_records_dates(target.entity, from_date, to_date + timedelta(seconds=1), 2 * df.size):
                new_value = df.get(event.datetime, False)
                if new_value is False:
                    # this existing event is not in the handled data
                    # this could indicate duplicate values
                    if any([e.datetime == event.datetime for e in events_update]):
                        events_delete.append(event)
                elif new_value is None:
                    # this indicates that we do not want an event in this location
                    events_delete.append(event)
                elif new_value == event.value:
                    # no changes
                    # remove event from creation
                    df = df.drop(event.datetime)
                elif new_value != event.value:
                    # remove event from creation
                    df = df.drop(event.datetime)
                    if self.che.handler_event_change == CHEventChange.UPDATE:
                        event.set_value(new_value)
                        events_update.append(event)
                if not isinstance(new_value, (float, int)):
                    print('new_value check', new_value, type(new_value))
        df = df.dropna()
        for k, v in df.to_dict().items():
            e = target.entity.create_event(value=v, dt=k)
            events_create.append(e)

        self.che.events_create_with_counter(events_create, log_tag=log_tag)
        self.che.events_update_with_counter(events_update, log_tag=log_tag)
        self.che.events_delete_with_counter(events_delete, log_tag=log_tag)

    def _duplicate_events(self, job: CustomJob, from_date: datetime, to_date: datetime) -> None:
        for t in self.che.targets:
            if t.job != job:
                continue
            # count events
            df = self.get_aggregation_df(
                source=CEDSource('label', CEDAggregation.COUNT),
                entity=t.entity,
                target=t,
                from_date=from_date,
                to_date=to_date
            )
            duplicates = df[df > 1]
            if duplicates.empty:
                continue
            log.info('%s duplicates for %s:\n%s',
                     duplicates.sum() - duplicates.count(),
                     t.entity,
                     duplicates)
            keep_events = {}
            delete_events = []
            for e in self.che.event_records_dates(
                entity=t.entity,
                from_date=duplicates.index.min,
                to_date=t.interval.shift_datetime(duplicates.index.max, -1),
                size=min(duplicates['count'].sum(), 1000)
            ):
                reference = keep_events.get(e.datetime)
                if reference is None:
                    keep_events.update({e.datetime: e})
                elif e.datetime_created > reference.datetime_created:
                    delete_events.append(reference)
                    keep_events.update({e.datetime: e})
                else:
                    delete_events.append(e)
            self.che.events_delete_with_counter(delete_events, log_tag=t.entity.public_id)

    def _purge_events(self, job: CustomJob, from_date: datetime, to_date: datetime) -> None:
        for t in self.che.targets:
            if t.job != job:
                continue
            # count events
            delete_events=[]
            for e in self.che.event_records_dates(
                entity=t.entity,
                from_date=from_date,
                to_date=to_date,
                size=1000
            ):
                delete_events.append(e)
            self.che.events_delete_with_counter(delete_events, log_tag=t.entity.public_id)