0

I'm using RxCpp to process a stream of order data and time slices. My goal is to segment the order data into windows, where each window is defined by three consecutive time slice messages. However, I'm encountering an issue where the order data within the windows is being duplicated, especially in later windows.

Problem Description

below is the source code for my program , the output of the program is behind of the code.

Issue

As seen in the output, the same orders are appearing multiple times within a single window, and this duplication increases with subsequent windows.What might be causing the duplication of data within the windows, and how can I modify the code to ensure each order appears only once in each window?

Any insights or suggestions would be greatly appreciated!

code

#include <rxcpp/rx.hpp>
#include <iostream>
#include <string>
#include <unordered_map>
#include <variant>
#include <chrono>
#include <thread>
#include <fstream>
using namespace std;
using namespace rxcpp;
using namespace rxcpp::operators;
using namespace std::chrono;


struct Order {
    string investorid;
    string instrumentid;
    string exchangeid;
    double amount;
    int transactionid;

    friend ostream& operator<<(ostream& os, const Order& order) {
        os << "Order(investorid: " << order.investorid
           << ", instrumentid: " << order.instrumentid
           << ", exchangeid: " << order.exchangeid
           << ", amount: " << order.amount
           << ", transactionid: " << order.transactionid << ")";
        return os;
    }

    friend bool operator==(const Order& order1, const Order& order2) {

        return order1.investorid == order2.investorid
            && order1.instrumentid == order2.instrumentid
            && order1.exchangeid == order2.exchangeid
            && order1.amount == order2.amount
            && order1.transactionid == order2.transactionid;
    }
};


struct TimeSlice {
    int64_t timestamp; // milliseconds since epoch

    friend ostream& operator<<(ostream& os, const TimeSlice& timeslice) {
        os << "TimeSlice(timestamp: " << timeslice.timestamp << ")";
        return os;
    }
       friend bool operator==(const TimeSlice& ts1, const TimeSlice& ts2) {
        return ts1.timestamp == ts2.timestamp;
    }
};

using OrderOrTimeSlice = variant<Order, TimeSlice>;
std::ostream& operator<<(std::ostream& os, const OrderOrTimeSlice& data) {
    std::visit([&](const auto& value) { os << value; }, data);
    return os;
}

int main() {

    std::ofstream mixedDataStreamFile("mixedDataStream.txt");
    std::ofstream timeWindowsDataFile("timeWindowsData.txt");



    auto orders = observable<>::interval(milliseconds(150)).take(100)
                  | rxcpp::operators::map([](int i) {
                        Order order;
                        order.investorid = "Investor" + to_string(i % 3 + 1);
                        order.instrumentid = "Instrument" + to_string(i % 5 + 1);
                        order.exchangeid = "Exchange" + to_string(i % 2 + 1);
                        order.amount = 1000.0 * i;
                        order.transactionid = i;
                        return OrderOrTimeSlice(order);
                    });

 
    auto time_slices = observable<>::interval(milliseconds(200)).take(100)
                       | rxcpp::operators::map([](int i){
                             TimeSlice timeslice;
                             i += 1;
                             timeslice.timestamp = duration_cast<milliseconds>(
                                                       system_clock::now().time_since_epoch()
                                                   ).count();
                             return OrderOrTimeSlice(timeslice);
                         });


    auto merged = orders.merge(time_slices);
    

  




    auto openings = merged | filter([](const OrderOrTimeSlice& ots) {
        return holds_alternative<TimeSlice>(ots);
    });


    auto closing_selector = [counter = make_shared<int>(0)](const OrderOrTimeSlice& ) mutable {
        return observable<>::create<OrderOrTimeSlice>([counter](subscriber<OrderOrTimeSlice> s) {
            *counter += 1;
            if (*counter % 3 == 0) {
                s.on_next(TimeSlice{});
                s.on_completed();
            }
        });
    };

    
    auto windows = merged|window_toggle(openings, closing_selector);

   
    int64_t timestampClose; // m
    int nWid = 0;
    windows|distinct_until_changed()|subscribe<observable<variant<Order, TimeSlice>> >(
            [&](observable<variant<Order, TimeSlice>> window) {
  
    
                window|distinct_until_changed()|subscribe<variant<Order, TimeSlice>>(
                    [&](const OrderOrTimeSlice& data) {
      
                    if (std::holds_alternative<Order>(data)) {
                        const Order& order = std::get<Order>(data);
                        timeWindowsDataFile << "Window id "<<nWid <<",Data: " << order<<std::endl ;
                        }
                   
                    else
                        {
                        const TimeSlice& order = std::get<TimeSlice>(data);
                        timestampClose = order.timestamp;
                        }
                    },
                   
                    [&]() {
                    timeWindowsDataFile << "Time Window id "<<nWid<<", "<< timestampClose<<" closed" << std::endl;
                    nWid++;
                    }
                );
    
            },
            [](std::exception_ptr) {

            }
        );
    

 
    this_thread::sleep_for(chrono::seconds(10));

  
    mixedDataStreamFile.close();
    timeWindowsDataFile.close();

    return 0;
}

program output:

Window id 0,Data: Order(investorid: Investor3, instrumentid: Instrument3, exchangeid: Exchange1, amount: 2000, transactionid: 2)
Window id 0,Data: Order(investorid: Investor1, instrumentid: Instrument4, exchangeid: Exchange2, amount: 3000, transactionid: 3)
Window id 0,Data: Order(investorid: Investor1, instrumentid: Instrument4, exchangeid: Exchange2, amount: 3000, transactionid: 3)
Time Window id 0, 1718190943365 closed
Window id 1,Data: Order(investorid: Investor2, instrumentid: Instrument5, exchangeid: Exchange1, amount: 4000, transactionid: 4)
Window id 1,Data: Order(investorid: Investor2, instrumentid: Instrument5, exchangeid: Exchange1, amount: 4000, transactionid: 4)
Window id 1,Data: Order(investorid: Investor3, instrumentid: Instrument1, exchangeid: Exchange2, amount: 5000, transactionid: 5)
Window id 1,Data: Order(investorid: Investor3, instrumentid: Instrument1, exchangeid: Exchange2, amount: 5000, transactionid: 5)
Window id 1,Data: Order(investorid: Investor1, instrumentid: Instrument2, exchangeid: Exchange1, amount: 6000, transactionid: 6)
Window id 1,Data: Order(investorid: Investor1, instrumentid: Instrument2, exchangeid: Exchange1, amount: 6000, transactionid: 6)
Window id 1,Data: Order(investorid: Investor1, instrumentid: Instrument2, exchangeid: Exchange1, amount: 6000, transactionid: 6)
Window id 1,Data: Order(investorid: Investor2, instrumentid: Instrument3, exchangeid: Exchange2, amount: 7000, transactionid: 7)
Window id 1,Data: Order(investorid: Investor2, instrumentid: Instrument3, exchangeid: Exchange2, amount: 7000, transactionid: 7)
Window id 1,Data: Order(investorid: Investor2, instrumentid: Instrument3, exchangeid: Exchange2, amount: 7000, transactionid: 7)
Window id 1,Data: Order(investorid: Investor2, instrumentid: Instrument3, exchangeid: Exchange2, amount: 7000, transactionid: 7)
Time Window id 1, 1718190943958 closed
Window id 2,Data: Order(investorid: Investor3, instrumentid: Instrument4, exchangeid: Exchange1, amount: 8000, transactionid: 8)
Window id 2,Data: Order(investorid: Investor3, instrumentid: Instrument4, exchangeid: Exchange1, amount: 8000, transactionid: 8)
Window id 2,Data: Order(investorid: Investor3, instrumentid: Instrument4, exchangeid: Exchange1, amount: 8000, transactionid: 8)
Window id 2,Data: Order(investorid: Investor3, instrumentid: Instrument4, exchangeid: Exchange1, amount: 8000, transactionid: 8)
Window id 2,Data: Order(investorid: Investor1, instrumentid: Instrument5, exchangeid: Exchange2, amount: 9000, transactionid: 9)
Window id 2,Data: Order(investorid: Investor1, instrumentid: Instrument5, exchangeid: Exchange2, amount: 9000, transactionid: 9)
Window id 2,Data: Order(investorid: Investor1, instrumentid: Instrument5, exchangeid: Exchange2, amount: 9000, transactionid: 9)
Window id 2,Data: Order(investorid: Investor1, instrumentid: Instrument5, exchangeid: Exchange2, amount: 9000, transactionid: 9)
Window id 2,Data: Order(investorid: Investor2, instrumentid: Instrument1, exchangeid: Exchange1, amount: 10000, transactionid: 10)
Window id 2,Data: Order(investorid: Investor2, instrumentid: Instrument1, exchangeid: Exchange1, amount: 10000, transactionid: 10)
Window id 2,Data: Order(investorid: Investor2, instrumentid: Instrument1, exchangeid: Exchange1, amount: 10000, transactionid: 10)
Window id 2,Data: Order(investorid: Investor2, instrumentid: Instrument1, exchangeid: Exchange1, amount: 10000, transactionid: 10)
Window id 2,Data: Order(investorid: Investor2, instrumentid: Instrument1, exchangeid: Exchange1, amount: 10000, transactionid: 10)
Window id 2,Data: Order(investorid: Investor3, instrumentid: Instrument2, exchangeid: Exchange2, amount: 11000, transactionid: 11)
Window id 2,Data: Order(investorid: Investor3, instrumentid: Instrument2, exchangeid: Exchange2, amount: 11000, transactionid: 11)
Window id 2,Data: Order(investorid: Investor3, instrumentid: Instrument2, exchangeid: Exchange2, amount: 11000, transactionid: 11)
Window id 2,Data: Order(investorid: Investor3, instrumentid: Instrument2, exchangeid: Exchange2, amount: 11000, transactionid: 11)
Window id 2,Data: Order(investorid: Investor3, instrumentid: Instrument2, exchangeid: Exchange2, amount: 11000, transactionid: 11)
Window id 2,Data: Order(investorid: Investor3, instrumentid: Instrument2, exchangeid: Exchange2, amount: 11000, transactionid: 11)
Time Window id 2, 1718190944553 closed
Window id 3,Data: Order(investorid: Investor1, instrumentid: Instrument3, exchangeid: Exchange1, amount: 12000, transactionid: 12)
Window id 3,Data: Order(investorid: Investor1, instrumentid: Instrument3, exchangeid: Exchange1, amount: 12000, transactionid: 12)
Window id 3,Data: Order(investorid: Investor1, instrumentid: Instrument3, exchangeid: Exchange1, amount: 12000, transactionid: 12)
Window id 3,Data: Order(investorid: Investor1, instrumentid: Instrument3, exchangeid: Exchange1, amount: 12000, transactionid: 12)
Window id 3,Data: Order(investorid: Investor1, instrumentid: Instrument3, exchangeid: Exchange1, amount: 12000, transactionid: 12)
Window id 3,Data: Order(investorid: Investor1, instrumentid: Instrument3, exchangeid: Exchange1, amount: 12000, transactionid: 12)
Window id 3,Data: Order(investorid: Investor2, instrumentid: Instrument4, exchangeid: Exchange2, amount: 13000, transactionid: 13)
Window id 3,Data: Order(investorid: Investor2, instrumentid: Instrument4, exchangeid: Exchange2, amount: 13000, transactionid: 13)
Window id 3,Data: Order(investorid: Investor2, instrumentid: Instrument4, exchangeid: Exchange2, amount: 13000, transactionid: 13)
Window id 3,Data: Order(investorid: Investor2, instrumentid: Instrument4, exchangeid: Exchange2, amount: 13000, transactionid: 13)
Window id 3,Data: Order(investorid: Investor2, instrumentid: Instrument4, exchangeid: Exchange2, amount: 13000, transactionid: 13)
Window id 3,Data: Order(investorid: Investor2, instrumentid: Instrument4, exchangeid: Exchange2, amount: 13000, transactionid: 13)
Window id 3,Data: Order(investorid: Investor3, instrumentid: Instrument5, exchangeid: Exchange1, amount: 14000, transactionid: 14)
Window id 3,Data: Order(investorid: Investor3, instrumentid: Instrument5, exchangeid: Exchange1, amount: 14000, transactionid: 14)
Window id 3,Data: Order(investorid: Investor3, instrumentid: Instrument5, exchangeid: Exchange1, amount: 14000, transactionid: 14)
Window id 3,Data: Order(investorid: Investor3, instrumentid: Instrument5, exchangeid: Exchange1, amount: 14000, transactionid: 14)
Window id 3,Data: Order(investorid: Investor3, instrumentid: Instrument5, exchangeid: Exchange1, amount: 14000, transactionid: 14)
Window id 3,Data: Order(investorid: Investor3, instrumentid: Instrument5, exchangeid: Exchange1, amount: 14000, transactionid: 14)
Window id 3,Data: Order(investorid: Investor3, instrumentid: Instrument5, exchangeid: Exchange1, amount: 14000, transactionid: 14)
Window id 3,Data: Order(investorid: Investor1, instrumentid: Instrument1, exchangeid: Exchange2, amount: 15000, transactionid: 15)
Window id 3,Data: Order(investorid: Investor1, instrumentid: Instrument1, exchangeid: Exchange2, amount: 15000, transactionid: 15)
Window id 3,Data: Order(investorid: Investor1, instrumentid: Instrument1, exchangeid: Exchange2, amount: 15000, transactionid: 15)
Window id 3,Data: Order(investorid: Investor1, instrumentid: Instrument1, exchangeid: Exchange2, amount: 15000, transactionid: 15)
Window id 3,Data: Order(investorid: Investor1, instrumentid: Instrument1, exchangeid: Exchange2, amount: 15000, transactionid: 15)
Window id 3,Data: Order(investorid: Investor1, instrumentid: Instrument1, exchangeid: Exchange2, amount: 15000, transactionid: 15)
Window id 3,Data: Order(investorid: Investor1, instrumentid: Instrument1, exchangeid: Exchange2, amount: 15000, transactionid: 15)
Window id 3,Data: Order(investorid: Investor1, instrumentid: Instrument1, exchangeid: Exchange2, amount: 15000, transactionid: 15)
Time Window id 3, 1718190945166 closed```



What might be causing the duplication of data within the windows, and how can I modify the code to ensure each order appears only once in each window?

Any insights or suggestions would be greatly appreciated!
Improve this question

1 Answer 1

Reset to default
0

I hope it is still relevant for you =)

I think, problem is there:

auto windows = merged|window_toggle(openings, closing_selector);

Actually it is

auto windows = merged|window_toggle(merged | filter(...), closing_selector);

so, you are subsribing to same merged twice. BUT your merged is simple merge of two simple observables. So, in this case you would subscribe to 2 identical merged observables and they would work in parallel. To avoid it you have to apply .publish().ref_count() to your merged observable: it would keep same observable and share it between different subscribers

Difference:

const auto observable = observable<>::interval(milliseconds(150)).take(100);

observable.subscribe([](int i) { std::cout << "#1 " << i << std::endl; });
observable.subscribe([](int i) { std::cout << "#2 " << i << std::endl; });

would output something like

#1 0
#1 1
#1 2
#1 3
#2 0 <----
#2 1
#1 4
#2 2

while

const auto observable = observable<>::interval(milliseconds(150)).take(100).publish().ref_count();

observable.subscribe([](int i) { std::cout << "#1 " << i << std::endl; });
observable.subscribe([](int i) { std::cout << "#2 " << i << std::endl; });

would attach second subscriber to same observable as first subscriber

#1 0
#1 1
#1 2
#1 3
#2 3 <----
#1 4
#2 4
Improve this answer

Not the answer you're looking for? Browse other questions tagged or ask your own question.