/*
 * Copyright (C) 2013 - Johannes Schauer <j.schauer at email.de>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation;
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 *
 * 
 * This code heavily borrows from ns3 itself which are copyright of their
 * respective authors and redistributable under the same conditions.
 *
 */

#include <stddef.h>                     // for size_t
#include <stdint.h>                     // for uint32_t
#include <stdlib.h>                     // for exit
#include <algorithm>                    // for min
#include <iomanip>                      // for operator<<, setw
#include <iostream>                     // for operator<<, basic_ostream, etc
#include <string>                       // for string, allocator, etc
#include <vector>                       // for vector
#include "ns3/address.h"                // for Address
#include "ns3/application-container.h"  // for ApplicationContainer
#include "ns3/application.h"            // for Application
#include "ns3/boolean.h"                // for BooleanValue
#include "ns3/callback.h"               // for MakeCallback
#include "ns3/command-line.h"           // for CommandLine
#include "ns3/config.h"                 // for SetDefault, Connect
#include "ns3/data-rate.h"              // for DataRate
#include "ns3/double.h"                 // for DoubleValue
#include "ns3/enum.h"                   // for EnumValue
#include "ns3/error-model.h"            // for RateErrorModel
#include "ns3/event-id.h"               // for EventId
#include "ns3/fatal-error.h"            // for NS_FATAL_ERROR
#include "ns3/global-value.h"           // for GlobalValue
#include "ns3/header.h"                 // for Header
#include "ns3/inet-socket-address.h"    // for InetSocketAddress
#include "ns3/internet-stack-helper.h"  // for InternetStackHelper
#include "ns3/ipv4-address-helper.h"    // for Ipv4AddressHelper
#include "ns3/ipv4-address.h"           // for operator<<, Ipv4Address, etc
#include "ns3/ipv4-global-routing-helper.h"
#include "ns3/ipv4-header.h"            // for Ipv4Header
#include "ns3/ipv4-interface-container.h"  // for Ipv4InterfaceContainer
#include "ns3/ipv4.h"                   // for Ipv4
#include "ns3/net-device-container.h"   // for NetDeviceContainer
#include "ns3/net-device.h"             // for NetDevice
#include "ns3/node-container.h"         // for NodeContainer
#include "ns3/node.h"                   // for Node
#include "ns3/nstime.h"                 // for Seconds, Time
#include "ns3/object.h"                 // for CreateObject
#include "ns3/packet-sink-helper.h"     // for PacketSinkHelper
#include "ns3/packet.h"                 // for Packet
#include "ns3/point-to-point-helper.h"  // for PointToPointHelper
#include "ns3/point-to-point-module.h"
#include "ns3/traffic-control-module.h"
#include "ns3/pointer.h"                // for PointerValue
#include "ns3/ptr.h"                    // for Ptr, Create, DynamicCast
#include "ns3/trace-helper.h"           // for ascii:CreateFileStream
#include "ns3/rng-seed-manager.h"       // for SeedManager
#include "ns3/simulator.h"              // for Simulator
#include "ns3/socket.h"                 // for Socket, etc
#include "ns3/string.h"                 // for StringValue
#include "ns3/tcp-header.h"             // for TcpHeader
#include "ns3/tcp-socket.h"             // for TcpHeader
#include "ns3/tcp-westwood.h"
#include "ns3/type-id.h"                // for TypeIdValue
#include "ns3/udp-header.h"             // for UdpHeader
#include "ns3/udp-socket-factory.h"     // for UdpSocketFactory
#include "ns3/uinteger.h"               // for UintegerValue
#include "ns3/drop-tail-queue.h"               // for UintegerValue

/*
 * 0 disabled, 1 works
 * Print whenever an IPv4 packet leaves a NetDevice and when it is received by a NetDevice.
 * Does not understand dropping.
 */
#define STACK_ASCII_TRACE 1

/*
 * 0 disabled, 1 works
 * reports all Enqueue, Dequeue and MacRx events for all leaf devices
 */
#define LEAF_NODE_ASCII_TRACE 1

/*
 * 0 disabled, 1 works
 * reports all Enqueue, Dequeue and MacRx events for all bottleneck devices
 */
#define BOTTLENECK_NODE_ASCII_TRACE 1

using namespace ns3;

struct Parameters
{
    std::string transport_prot;
    double      error_p;
    std::string bottleneck_bandwidth;
    std::string access_bandwidth;
    std::string bottleneck_delay;
    std::string access_delay;
    uint32_t    queuesize;
    uint32_t    run;
    double      simstop;
    std::string ostream_prefix;

    const bool  trace_cwnd;
    const bool  trace_ssthresh;
    const bool  trace_rtt;
    const bool  trace_packets_in_queue;
    const bool  trace_bytes_in_queue;
    const bool  trace_time_in_queue;        /* this trace does not exist */
    const bool  trace_packets_in_qdisc;     /* packets currently in qdisc - not useful */
    const bool  trace_bytes_in_qdisc;       /* bytes currently in qdisc - not useful */
    const bool  trace_time_in_qdisc;        /* time last packet spent in qdisc - not useful */
    const bool  trace_drop_in_qdisc;
    const bool  trace_packets;

    std::ofstream trace_cwnd_stream;
    std::ofstream trace_ssthresh_stream;
    std::ofstream trace_rtt_stream;
    std::ofstream trace_packets_in_queue_stream;
    std::ofstream trace_bytes_in_queue_stream;
    std::ofstream trace_time_in_queue_stream;
    std::ofstream trace_packets_in_qdisc_stream;
    std::ofstream trace_bytes_in_qdisc_stream;
    std::ofstream trace_time_in_qdisc_stream;
    std::ofstream trace_drop_in_qdisc_stream;

    Parameters()
        : trace_cwnd            ( true )
        , trace_ssthresh        ( true )
        , trace_rtt             ( true )
        , trace_packets_in_queue( true )
        , trace_bytes_in_queue  ( true )
        , trace_time_in_queue   ( false )
        , trace_packets_in_qdisc( false )
        , trace_bytes_in_qdisc  ( false )
        , trace_time_in_qdisc   ( false )
        , trace_drop_in_qdisc   ( true )
        , trace_packets         ( true )
    {
        transport_prot       = "TcpNewReno";
        error_p              = 0.0;
        bottleneck_bandwidth = "1Mbps";
        access_bandwidth     = "10Mbps";
        bottleneck_delay     = "50ms";
        access_delay         = "1ms";
        queuesize            = 90;
        run                  = 0;
        simstop              = 100.0;

        ostream_prefix       = "";
    }

    void parse(int argc, char **argv)
    {
        ns3::CommandLine cmd;
        cmd.AddValue("tcpcaa",
                "Default TCP congestion-avoidance algorithm to use: "
                "TcpNewReno, TcpVegas, TcpCubic, TcpBbr",
                transport_prot);
        cmd.AddValue("error_p", "Packet error rate", error_p);
        cmd.AddValue("bandwidth", "Bottleneck link bandwidth",
                bottleneck_bandwidth);
        cmd.AddValue("access_bandwidth", "Access link bandwidth",
                access_bandwidth);
        cmd.AddValue("delay", "Bottleneck link delay", bottleneck_delay);
        cmd.AddValue("access delay", "Access link delay", access_delay);
        cmd.AddValue("run", "Run index (for setting repeatable seeds)", run);
        cmd.AddValue("simstop", "Stop simulator after this many seconds",
                simstop);
        cmd.AddValue("queue_size",
                "Maximum amount of packets allowed inside the queue", queuesize);
        cmd.Parse(argc, argv);

        ostream_prefix = transport_prot + "-qlen-" + std::to_string(queuesize) + "-bw-" + bottleneck_bandwidth + "-delay-" + bottleneck_delay + "-";

        if( trace_cwnd )
            trace_cwnd_stream.open( ostream_prefix + "cwnd.tr" );

        if( trace_ssthresh )
            trace_ssthresh_stream.open( ostream_prefix + "ssthresh.tr" );

        if( trace_rtt )
            trace_rtt_stream.open( ostream_prefix + "rtt.tr" );

        if( trace_packets_in_queue )
            trace_packets_in_queue_stream.open( ostream_prefix + "qlen-pkt.tr" );

        if( trace_bytes_in_queue )
            trace_bytes_in_queue_stream.open( ostream_prefix + "qlen-byte.tr" );

        if( trace_time_in_queue )
            trace_time_in_queue_stream.open( ostream_prefix + "qlen-time.tr" );

        if( trace_packets_in_qdisc )
            trace_packets_in_qdisc_stream.open( ostream_prefix + "qdisclen-pkt.tr" );

        if( trace_bytes_in_qdisc )
            trace_bytes_in_qdisc_stream.open( ostream_prefix + "qdisclen-byte.tr" );

        if( trace_time_in_qdisc )
            trace_time_in_qdisc_stream.open( ostream_prefix + "qdisclen-time.tr" );

        if( trace_drop_in_qdisc )
            trace_drop_in_qdisc_stream.open( ostream_prefix + "qdisclen-drop.tr" );

        ns3::SeedManager::SetSeed(1);
        ns3::SeedManager::SetRun(run);
    }

    void configure()
    {
        /*
         * Setting basic TCP socket types best on command line parameter.
         */
        if (transport_prot.compare("TcpWestwood") == 0)
        {
            Config::SetDefault( "ns3::TcpL4Protocol::SocketType",
                                TypeIdValue(TcpWestwood::GetTypeId()) );
            Config::SetDefault( "ns3::TcpWestwood::ProtocolType",
                                EnumValue (TcpWestwood::WESTWOODPLUS) );
            // ns3::Config::SetDefault("ns3::TcpWestwood::FilterType",
            //                         ns3::EnumValue(ns3::TcpWestwood::TUSTIN));
        }
        else
        {
            transport_prot = std::string ("ns3::") + transport_prot;
            TypeId tcpTid;
            NS_ABORT_MSG_UNLESS( TypeId::LookupByNameFailSafe( transport_prot, &tcpTid ),
                                 "TypeId " << transport_prot << " not found" );
            Config::SetDefault( "ns3::TcpL4Protocol::SocketType",
                                TypeIdValue( TypeId::LookupByName( transport_prot) ) );
        }
    }
};

Parameters param;

/*
 * we need to roll our own app instead of using one of the supplied helpers
 * because of the following reasons:
 *
 *  - bulktransferhelper can't do udp
 *  - onoffhelper doesnt allow access to the underlying sockets
 *  - not having access to the underlying sockets, means that any tracer can
 *    only be registered *after* the app created by the helper is started.
 *    This is ugly as it forces to start multiple tracers manually in synch
 *    with the app start time. Since it has to be multiple tracers, one cannot
 *    anymore use the wildcard mechanism to have a one-tracer-catches-all
 *    setup.
 *
 * this app allows to send an infinite number of tcp or udp packets at a fixed
 * rate
 * */
class LimitedRateApp : public ns3::Application
{
    public:

        LimitedRateApp();
        virtual ~ LimitedRateApp();

        void Setup(ns3::Ptr<ns3::Socket> socket, ns3::Address address,
                uint32_t packetSize, ns3::DataRate dataRate);

    private:
        virtual void StartApplication(void);
        virtual void StopApplication(void);

        void ScheduleTx(void);
        void SendPacket(void);

        ns3::Ptr<ns3::Socket> m_socket;
        ns3::Address m_peer;
        uint32_t m_packetSize;
        ns3::DataRate m_dataRate;
        ns3::EventId m_sendEvent;
        bool m_running;
};

LimitedRateApp::LimitedRateApp()
  : m_socket(0),
    m_peer(),
    m_packetSize(0),
    m_dataRate(0),
    m_sendEvent(),
    m_running(false)
{
}

LimitedRateApp::~LimitedRateApp()
{
    m_socket = 0;
}

    void
LimitedRateApp::Setup(ns3::Ptr<ns3::Socket> socket,
        ns3::Address address, uint32_t packetSize,
        ns3::DataRate dataRate)
{
    m_socket = socket;
    m_peer = address;
    m_packetSize = packetSize;
    m_dataRate = dataRate;
}

void LimitedRateApp::StartApplication(void)
{
    m_running = true;
    m_socket->Bind();
    m_socket->Connect(m_peer);
    SendPacket();
}

void LimitedRateApp::StopApplication(void)
{
    m_running = false;

    if (m_sendEvent.IsRunning()) {
        ns3::Simulator::Cancel(m_sendEvent);
    }

    if (m_socket) {
        m_socket->Close();
    }
}

void LimitedRateApp::SendPacket(void)
{
    ns3::Ptr<ns3::Packet> packet =
        ns3::Create<ns3::Packet>(m_packetSize);
    m_socket->Send(packet);
    ScheduleTx();
}

void LimitedRateApp::ScheduleTx(void)
{
    if (m_running) {
        ns3::Time tNext(ns3::Seconds(
                    m_packetSize * 8 /
                    static_cast<double>(m_dataRate.GetBitRate())));
        m_sendEvent = ns3::Simulator::Schedule(
                tNext, &LimitedRateApp::SendPacket, this);
    }
}

/*
 * this app borrows from the BulkSendApplication but keeps sending for an
 * infinite time (until Stop() is reached) and receives an already created
 * socket instead of creating its own
 *
 * it has unlimited rate in the sense that it pushes new packets into the
 * sending buffer of the socket until it is full and keeps filling that buffer
 * once the socket becomes ready to send again.
 *
 * due to this nature of operation it is only suitable for streaming sockets
 */
class UnlimitedRateApp : public ns3::Application
{
    public:
        UnlimitedRateApp();
        virtual ~ UnlimitedRateApp();

        void Setup(ns3::Ptr<ns3::Socket> socket, ns3::Address address,
                uint32_t packetSize);

    private:
        // inherited from Application base class.
        virtual void StartApplication(void);        // Called at time specified by Start
        virtual void StopApplication(void); // Called at time specified by Stop

        void SendData();

        ns3::Ptr<ns3::Socket> m_socket;  // Associated socket
        ns3::Address m_peer;        // Peer address
        bool m_connected;           // True if connected
        uint32_t m_packetSize;      // Size of data to send each time

        void ConnectionSucceeded(ns3::Ptr<ns3::Socket> socket);
        void ConnectionFailed(ns3::Ptr<ns3::Socket> socket);
        void DataSend(ns3::Ptr<ns3::Socket>, uint32_t);  // for socket's SetSendCallback
        void Ignore(ns3::Ptr<ns3::Socket> socket);
};

UnlimitedRateApp::UnlimitedRateApp()
 :  m_socket(0),
    m_connected(false),
    m_packetSize(512)
{
}

UnlimitedRateApp::~UnlimitedRateApp()
{
    m_socket = 0;
}

    void
UnlimitedRateApp::Setup(ns3::Ptr<ns3::Socket> socket,
        ns3::Address address, uint32_t packetSize)
{
    m_socket = socket;
    m_peer = address;
    m_packetSize = packetSize;

    // Fatal error if socket type is not NS3_SOCK_STREAM or NS3_SOCK_SEQPACKET
    if (m_socket->GetSocketType() != ns3::Socket::NS3_SOCK_STREAM &&
            m_socket->GetSocketType() != ns3::Socket::NS3_SOCK_SEQPACKET) {
        NS_FATAL_ERROR
            ("Using UnlimitedRateApp with an incompatible socket type. "
             "UnlimitedRateApp requires SOCK_STREAM or SOCK_SEQPACKET. "
             "In other words, use TCP instead of UDP.");
    }
}

void UnlimitedRateApp::StartApplication(void)
{
    m_socket->Bind();
    m_socket->Connect(m_peer);
    m_socket->ShutdownRecv();
    m_socket->SetConnectCallback(ns3::MakeCallback(
                &UnlimitedRateApp::ConnectionSucceeded, this),
            ns3::MakeCallback(
                &UnlimitedRateApp::ConnectionFailed, this));
    m_socket->SetSendCallback(ns3::MakeCallback(
                &UnlimitedRateApp::DataSend, this));
    if (m_connected)
        SendData();
}

void UnlimitedRateApp::StopApplication(void)    // Called at time specified by Stop
{
    if (m_socket != 0) {
        m_socket->Close();
        m_connected = false;
    }
}

void UnlimitedRateApp::SendData(void)
{
    // We exit this loop when actual<toSend as the send side
    // buffer is full. The "DataSend" callback will pop when
    // some buffer space has freed ip.
    for (;;) {
        ns3::Ptr<ns3::Packet> packet =
            ns3::Create<ns3::Packet>(m_packetSize);
        int actual = m_socket->Send(packet);
        if ((unsigned) actual != m_packetSize)
            break;
    }
}

void UnlimitedRateApp::ConnectionSucceeded(ns3::Ptr<ns3::Socket> socket)
{
    m_connected = true;
    SendData();
}

void UnlimitedRateApp::ConnectionFailed(ns3::Ptr<ns3::Socket> socket)
{
}

void UnlimitedRateApp::DataSend(ns3::Ptr<ns3::Socket>, uint32_t)
{
    if (m_connected) {          // Only send new data if the connection has completed
        ns3::Simulator::ScheduleNow(&UnlimitedRateApp::SendData, this);
    }
}


/*
 * this app borrows from the BulkSendApplication but instead allows a socket
 * being passed to it instead of creating its own socket internally.
 *
 * it has unlimited rate in the sense that it pushes new packets into the
 * sending buffer of the socket until it is full and keeps filling that buffer
 * once the socket becomes ready to send again.
 *
 * due to this nature of operation it is only suitable for streaming sockets
 */
class LimitedTransferApp : public ns3::Application
{
    public:
        LimitedTransferApp();
        virtual ~ LimitedTransferApp();

        void Setup(ns3::Ptr<ns3::Socket> socket, ns3::Address address,
                uint32_t packetSize, uint32_t maxBytes);

    private:
        // inherited from Application base class.
        virtual void StartApplication(void);        // Called at time specified by Start
        virtual void StopApplication(void); // Called at time specified by Stop

        void SendData();

        ns3::Ptr<ns3::Socket> m_socket;  // Associated socket
        ns3::Address m_peer;        // Peer address
        bool m_connected;           // True if connected
        uint32_t m_packetSize;      // Size of data to send each time
        uint32_t m_totBytes;        // Total bytes sent so far
        uint32_t m_maxBytes;        // Limit total number of bytes sent

        void ConnectionSucceeded(ns3::Ptr<ns3::Socket> socket);
        void ConnectionFailed(ns3::Ptr<ns3::Socket> socket);
        void DataSend(ns3::Ptr<ns3::Socket>, uint32_t);  // for socket's SetSendCallback
        void Ignore(ns3::Ptr<ns3::Socket> socket);
};

LimitedTransferApp::LimitedTransferApp()
  : m_socket(0),
    m_connected(false),
    m_packetSize(512),
    m_totBytes(0),
    m_maxBytes(0)
{
}

LimitedTransferApp::~LimitedTransferApp()
{
    m_socket = 0;
}

    void
LimitedTransferApp::Setup(ns3::Ptr<ns3::Socket> socket,
        ns3::Address address, uint32_t packetSize,
        uint32_t maxBytes)
{
    m_socket = socket;
    m_peer = address;
    m_packetSize = packetSize;
    m_maxBytes = maxBytes;

    // Fatal error if socket type is not NS3_SOCK_STREAM or NS3_SOCK_SEQPACKET
    if (m_socket->GetSocketType() != ns3::Socket::NS3_SOCK_STREAM &&
            m_socket->GetSocketType() != ns3::Socket::NS3_SOCK_SEQPACKET) {
        NS_FATAL_ERROR
            ("Using LimitedTransferApp with an incompatible socket type. "
             "LimitedTransferApp requires SOCK_STREAM or SOCK_SEQPACKET. "
             "In other words, use TCP instead of UDP.");
    }
}

void LimitedTransferApp::StartApplication(void)
{
    m_socket->Bind();
    m_socket->Connect(m_peer);
    m_socket->ShutdownRecv();
    m_socket->SetConnectCallback(MakeCallback(
                &LimitedTransferApp::ConnectionSucceeded, this),
            MakeCallback(
                &LimitedTransferApp::ConnectionFailed, this));
    m_socket->SetSendCallback(
            ns3::MakeCallback(&LimitedTransferApp::DataSend, this));
    if (m_connected)
        SendData();
}

void LimitedTransferApp::StopApplication(void)  // Called at time specified by Stop
{
    if (m_socket != 0) {
        m_socket->Close();
        m_connected = false;
    }
}

void LimitedTransferApp::SendData(void)
{
    while (m_totBytes<m_maxBytes) {
        uint32_t toSend = m_packetSize;
        // Make sure we don't send too many
        if (m_maxBytes> 0) {
            toSend = std::min(m_packetSize, m_maxBytes - m_totBytes);
        }
        ns3::Ptr<ns3::Packet> packet =
            ns3::Create<ns3::Packet>(toSend);
        int actual = m_socket->Send(packet);
        if (actual> 0) {
            m_totBytes += actual;
        }
        // We exit this loop when actual<toSend as the send side
        // buffer is full. The "DataSent" callback will pop when
        // some buffer space has freed ip.
        if ((unsigned) actual != toSend) {
            break;
        }
    }
    // Check if time to close (all sent)
    if (m_totBytes == m_maxBytes && m_connected) {
        m_socket->Close();
        m_connected = false;
    }
}

void LimitedTransferApp::ConnectionSucceeded(ns3::Ptr<ns3::Socket>
        socket)
{
    m_connected = true;
    SendData();
}

void LimitedTransferApp::ConnectionFailed(ns3::Ptr<ns3::Socket> socket)
{
}

void LimitedTransferApp::DataSend(ns3::Ptr<ns3::Socket>, uint32_t)
{
    if (m_connected) {          // Only send new data if the connection has completed
        ns3::Simulator::ScheduleNow(&LimitedTransferApp::SendData, this);
    }
}

static void CwndTracer(std::string context, uint32_t oldval,
        uint32_t newval)
{
    if (newval> 2147483648) {
        std::cerr << "impossibly high cwnd value: " << newval << std::endl;
        return;
    }
    std::ostream& o = param.trace_cwnd_stream;
    o << std::setw(8) << ns3::Simulator::Now().GetSeconds() << " ";
    o << context << " " << newval << std::endl;
}

static void SsThreshTracer(std::string context, uint32_t oldval,
        uint32_t newval)
{
    std::ostream& o = param.trace_ssthresh_stream;
    o << std::setw(8) << ns3::Simulator::Now().GetSeconds() << " ";
    o << context << " " << newval << std::endl;
}

static void RttTracer(std::string context, ns3::Time a, ns3::Time b )
{
    std::ostream& o = param.trace_rtt_stream;
    o << std::setw(8) << ns3::Simulator::Now().GetSeconds() << " ";
    o << context << " " << a.As(Time::MS) << " " << b.As(Time::MS) << std::endl;
}

#if 0
static void PacketSinkRxTracer(std::string context,
        ns3::Ptr<const ns3::Packet> p,
        const ns3::Address & addr)
{
    std::cout << std::setw(8) << ns3::Simulator::Now().GetSeconds() << " ";
    std::cout << context << " " << p->GetSize() << std::endl;
}
#endif

static void DevicePacketsInQueueTrace (uint32_t oldValue, uint32_t newValue)
{
    std::ostream& o = param.trace_packets_in_queue_stream;
    o << std::setw(8) << ns3::Simulator::Now().GetSeconds() << " ";
    o << "DevicePacketsInQueue " << oldValue << " to " << newValue << std::endl;
}
static void DeviceBytesInQueueTrace (uint32_t oldValue, uint32_t newValue)
{
    std::ostream& o = param.trace_bytes_in_queue_stream;
    o << std::setw(8) << ns3::Simulator::Now().GetSeconds() << " ";
    o << "DeviceBytesInQueue " << oldValue << " to " << newValue << std::endl;
}
static void DeviceTimeInQueueTrace( ns3::Time t )
{
    std::ostream& o = param.trace_time_in_queue_stream;
    o << std::setw(8) << ns3::Simulator::Now().GetSeconds() << " ";
    o << "SojournTime " << t.As(Time::MS) << std::endl;
}
static void DeviceBytesInQueueDiscTrace( unsigned int before, unsigned int now )
{
    std::ostream& o = param.trace_bytes_in_qdisc_stream;
    o << std::setw(8) << ns3::Simulator::Now().GetSeconds() << " ";
    o << "Bytes in queue: " << before << "->" << now << std::endl;
}
static void DevicePacketsInQueueDiscTrace( unsigned int before, unsigned int now )
{
    std::ostream& o = param.trace_packets_in_qdisc_stream;
    o << std::setw(8) << ns3::Simulator::Now().GetSeconds() << " ";
    o << "Packets in queue: " << before << "->" << now << std::endl;
}
static void DeviceTimeInQueueDiscTrace(ns3::Time t)
{
    std::ostream& o = param.trace_time_in_qdisc_stream;
    o << std::setw(8) << ns3::Simulator::Now().GetSeconds() << " ";
    o << "Sojourn time pkt " << t.As(Time::MS) << std::endl;
}

static void DevicePacketsInQueueDiscDrop (ns3::Ptr<const ns3::QueueDiscItem> p)
{
    std::ostream& o = param.trace_drop_in_qdisc_stream;
    o << std::setw(8) << ns3::Simulator::Now().GetSeconds() << " ";
    o << "Dropped pkt ";
    p->Print( o );
    o << std::endl;
}

int main(int argc, char *argv[])
{
    /*************************************************************
     * Phase I
     *
     * Parsing inputs and prepare parameters
     *************************************************************/
    param.parse( argc, argv );

    param.configure( );

    // read in traffic definitions from standard input
#if 0
    std::cerr << "Reading traffic descriptions from standard input..."
        << std::endl;
#endif
    std::vector<std::string> inputlines;
#if 0
    std::string input_line;
    for (std::string s; std::getline(std::cin, s);) {
        inputlines.push_back(s);
    }
#else
    inputlines.push_back("UR TCP 1024");
    inputlines.push_back("UR TCP 1024");
    inputlines.push_back("UR TCP 1024");
    inputlines.push_back("UR TCP 1024");
    inputlines.push_back("UR TCP 1024");
    inputlines.push_back("UR TCP 1024");
    inputlines.push_back("UR TCP 1024");
    inputlines.push_back("UR TCP 1024");
    inputlines.push_back("UR TCP 1024");
    inputlines.push_back("UR TCP 1024");
#endif

    uint32_t number_of_clients = inputlines.size();
    std::cerr << "creating " << number_of_clients << " data streams" << std::endl;

    /*************************************************************
     * Phase II
     *
     * Create the objects representing the network topology and
     * the entire network stack including applications.
     *
     * Setup of tracing output only where waiting for phase III
     * would be highly inconvenient.
     *************************************************************/

    /* Set default properties for all network links
     * that we are going to create.
     */
    ns3::PointToPointHelper bottleNeckLink;
    bottleNeckLink.SetDeviceAttribute( "DataRate",
                                       ns3::StringValue(param.bottleneck_bandwidth) );
    bottleNeckLink.SetChannelAttribute( "Delay",
                                        ns3::StringValue(param.bottleneck_delay) );
    bottleNeckLink.SetQueue( "ns3::DropTailQueue",
                             "MaxSize",
                             StringValue( std::to_string(param.queuesize) + "p") );

    ns3::PointToPointHelper pointToPointLeaf;
    pointToPointLeaf.SetDeviceAttribute( "DataRate",
                                         ns3::StringValue(param.access_bandwidth) );
    pointToPointLeaf.SetChannelAttribute( "Delay",
                                          ns3::StringValue(param.access_delay) );

    const int port = 9000;

    /* Create all nodes: 2 routers at both ends of the network,
     *                   number_of_clients senders, and
     *                   number_of_clients receivers
     */
    ns3::NodeContainer routers;
    ns3::NodeContainer leftleaves;
    ns3::NodeContainer rightleaves;

    routers    .Create(2);
    leftleaves .Create(number_of_clients);
    rightleaves.Create(number_of_clients);

    /* Create a link between the 2 routers with the default
     * properties specified above.
     * As a side effect, we get a NetDeviceContainer that contains
     * the 2 NetDevice, representing the network cards in the two
     * routers.
     */
    ns3::NetDeviceContainer routerdevices =
        bottleNeckLink.Install(routers);

    /* Do the same for every sender (left side).
     * Keep the object representing the network cards in NetDeviceContainers
     * for our convenience.
     */
    ns3::NetDeviceContainer leftrouterdevices;
    ns3::NetDeviceContainer leftleafdevices;

    for (uint32_t i = 0; i<number_of_clients; ++i)
    {
        ns3::NetDeviceContainer cleft =
            pointToPointLeaf.Install(routers.Get(0), leftleaves.Get(i));
        leftrouterdevices.Add(cleft.Get(0));
        leftleafdevices.Add(cleft.Get(1));
    }

    /* Do the same for every receiver (right side).
     * Keep the object representing the network cards in NetDeviceContainers
     * for our convenience.
     */
    ns3::NetDeviceContainer rightrouterdevices;
    ns3::NetDeviceContainer rightleafdevices;

    for (uint32_t i = 0; i<number_of_clients; ++i)
    {
        // add the right side links
        ns3::NetDeviceContainer cright =
            pointToPointLeaf.Install(routers.Get(1), rightleaves.Get(i));
        rightrouterdevices.Add(cright.Get(0));
        rightleafdevices.Add(cright.Get(1));
    }

    /* If we want to introduce random errors, do that by adding
     * random packet errors (Poisson-distributed) on all network
     * on the router side that connect to the senders and receivers.
     * These errors affect packets traveling from senders (data packets)
     * and from receivers (ACK packets) to the bottleneck link.
     * They are dropped on access link reception, which means that they
     * are never going into the bottleneck link's queue.
     */
    if( param.error_p != 0 )
    {
        // error model
        ns3::Ptr<ns3::RateErrorModel> em = ns3::CreateObject<ns3::RateErrorModel>();
        em->SetAttribute( "ErrorRate", ns3::DoubleValue( param.error_p ) );

        for (uint32_t i = 0; i<number_of_clients; ++i) {
            ns3::NetDeviceContainer cleft  = leftrouterdevices.Get(i);
            ns3::NetDeviceContainer cright = rightrouterdevices.Get(i);
            cleft.Get(0)->SetAttribute("ReceiveErrorModel",
                                       ns3::PointerValue(em));
            cright.Get(0)->SetAttribute("ReceiveErrorModel",
                                        ns3::PointerValue(em));
        }
    }

    /* Now add IP stacks to all end nodes as well as routers.
     * Assign them IP addresses and populate the routing tables.
     */
    // install internet stack on all nodes
    ns3::InternetStackHelper stack;
    stack.Install(routers);
    stack.Install(leftleaves);
    stack.Install(rightleaves);

    // assign ipv4 addresses (ipv6 addresses apparently are still not fully
    // supported by ns3)
    ns3::Ipv4AddressHelper routerips =
        ns3::Ipv4AddressHelper("10.3.0.0", "255.255.255.0");
    ns3::Ipv4AddressHelper leftips =
        ns3::Ipv4AddressHelper("10.1.0.0", "255.255.255.0");
    ns3::Ipv4AddressHelper rightips =
        ns3::Ipv4AddressHelper("10.2.0.0", "255.255.255.0");

    ns3::Ipv4InterfaceContainer routerifs;
    ns3::Ipv4InterfaceContainer leftleafifs;
    ns3::Ipv4InterfaceContainer leftrouterifs;
    ns3::Ipv4InterfaceContainer rightleafifs;
    ns3::Ipv4InterfaceContainer rightrouterifs;

    // assign addresses to connection connecting routers
    routerifs = routerips.Assign(routerdevices);

    // assign addresses to connection between routers and leaves
    for (uint32_t i = 0; i<number_of_clients; ++i) {
        // Assign to left side
        ns3::NetDeviceContainer ndcleft;
        ndcleft.Add(leftleafdevices.Get(i));
        ndcleft.Add(leftrouterdevices.Get(i));
        ns3::Ipv4InterfaceContainer ifcleft = leftips.Assign(ndcleft);
        leftleafifs.Add(ifcleft.Get(0));
        leftrouterifs.Add(ifcleft.Get(1));
        leftips.NewNetwork();
        // Assign to right side
        ns3::NetDeviceContainer ndcright;
        ndcright.Add(rightleafdevices.Get(i));
        ndcright.Add(rightrouterdevices.Get(i));
        ns3::Ipv4InterfaceContainer ifcright = rightips.Assign(ndcright);
        rightleafifs.Add(ifcright.Get(0));
        rightrouterifs.Add(ifcright.Get(1));
        rightips.NewNetwork();
    }

    ns3::Ipv4GlobalRoutingHelper::PopulateRoutingTables();

    /* Add applications on top of the simulated transport layer,
     * following the specification in the input.
     */
    ns3::ApplicationContainer sinkApps, udpApp;
    ns3::Address sinkLocalAddress(
            ns3::InetSocketAddress(
                ns3::Ipv4Address::GetAny(), port));
    ns3::PacketSinkHelper TcpPacketSinkHelper("ns3::TcpSocketFactory",
            sinkLocalAddress);
    ns3::PacketSinkHelper UdpPacketSinkHelper("ns3::UdpSocketFactory",
            sinkLocalAddress);

    // create all the source and sink apps
    for (size_t i = 0; i<inputlines.size(); ++i) {
        ns3::Ptr<ns3::Socket> sockptr;
        unsigned int pkgsize;
        float start;

        std::stringstream ss(inputlines[i]);
        std::string app, transport;
        ss >> app >> transport >> pkgsize >> start;

        if (transport.compare("TCP") == 0)
        {
            // setup source socket
            sockptr =
                ns3::Socket::CreateSocket(leftleaves.Get(i),
                        ns3::TcpSocketFactory::GetTypeId());
            ns3::Ptr<ns3::TcpSocket> tcpsockptr =
                ns3::DynamicCast<ns3::TcpSocket> (sockptr);
            tcpsockptr->SetAttribute("SegmentSize",
                    ns3::UintegerValue(pkgsize));
            std::stringstream nodeidss;
            nodeidss << leftleaves.Get(i)->GetId();
            std::string prefix = "/NodeList/" + nodeidss.str();

            // setup sink
            sinkApps.Add(TcpPacketSinkHelper.Install(rightleaves.Get(i)));
        }
        else if (transport.compare("UDP") == 0)
        {
            // setup source socket
            sockptr =
                ns3::Socket::CreateSocket(leftleaves.Get(i),
                        ns3::UdpSocketFactory::GetTypeId());
            // setup sink
            sinkApps.Add(UdpPacketSinkHelper.Install(rightleaves.Get(i)));
        }
        else
        {
            std::cerr << "unknown transport type: " <<
                transport << std::endl;
            exit(1);
        }

        if (app.compare("LR") == 0) {
            /* additionally read stop time and rate */
            float stop;
            std::string rate;
            ss >> stop >> rate;
            ns3::Ptr<LimitedRateApp> app =
                ns3::CreateObject<LimitedRateApp> ();
            app->Setup(sockptr,
                    ns3::InetSocketAddress(rightleafifs.GetAddress(i),
                        port), pkgsize,
                    ns3::DataRate(rate));
            leftleaves.Get(i)->AddApplication(app);
            app->SetStartTime(ns3::Seconds(start));
            app->SetStopTime(ns3::Seconds(stop));
        } else if (app.compare("UR") == 0) {
            /* additionally read stop time */
            float stop;
            ss >> stop;
            ns3::Ptr<UnlimitedRateApp> app =
                ns3::CreateObject<UnlimitedRateApp> ();
            app->Setup(sockptr,
                    ns3::InetSocketAddress(rightleafifs.GetAddress(i),
                        port), pkgsize);
            leftleaves.Get(i)->AddApplication(app);
            app->SetStartTime(ns3::Seconds(start));
            app->SetStopTime(ns3::Seconds(stop));
        } else if (app.compare("LT") == 0) {
            /* additionally read maxtransfer */
            unsigned int maxtransfer;
            ss >> maxtransfer;
            ns3::Ptr<LimitedTransferApp> app =
                ns3::CreateObject<LimitedTransferApp> ();
            app->Setup(sockptr,
                    ns3::InetSocketAddress(rightleafifs.GetAddress(i),
                        port), pkgsize, maxtransfer);
            leftleaves.Get(i)->AddApplication(app);
            app->SetStartTime(ns3::Seconds(start));
        } else {
            std::cerr << "unknown app type: " << app << std::endl;
            exit(1);
        }
    }

    /*************************************************************
     * Phase III
     *
     * Setup tracing
     *************************************************************/

    /* Add trace callbacks to the QueueDisc on the left side of
     * the bottleneck link.
     */
    if( param.trace_packets_in_qdisc ||
        param.trace_bytes_in_qdisc ||
        param.trace_time_in_qdisc ||
        param.trace_drop_in_qdisc )
    {
        Ptr<TrafficControlLayer> tc = routers.Get(0)->GetObject<TrafficControlLayer>();
        if( tc != 0 )
        {
            Ptr<QueueDisc> qd = tc->GetRootQueueDiscOnDevice( routerdevices.Get(0) );
            if( qd != 0 )
            {
                /* This does not give you the number of bytes in the queue, but only the
                 * number of packets that are processes by the QueueDisc.
                 * For our case you're only getting 0 or 1.
                 */
                if( param.trace_packets_in_qdisc )
                    qd->TraceConnectWithoutContext( "PacketsInQueue",
                                                    MakeCallback(&DevicePacketsInQueueDiscTrace) );

                /* This does not give you the number of bytes in the queue, but only the
                 * number of bytes that are processes by the QueueDisc.
                 */
                if( param.trace_bytes_in_qdisc )
                    qd->TraceConnectWithoutContext( "BytesInQueue",
                                                    MakeCallback(&DeviceBytesInQueueDiscTrace) );

                /* This appears to be the time between packet arrival and processing of
                 * a packet; the time a packet has spend inside the network stack before
                 * being handed off to the queue itself.
                 * The seems like a strange concept, at least in a simulation.
                 */
                if( param.trace_time_in_qdisc )
                    qd->TraceConnectWithoutContext( "SojournTime",
                                                    MakeCallback(&DeviceTimeInQueueDiscTrace) );

                /* This is where packets are actually getting lost due to congestion.
                 * An important trace source.
                 */
                if( param.trace_drop_in_qdisc )
                    qd->TraceConnectWithoutContext( "Drop",
                                                    MakeCallback(&DevicePacketsInQueueDiscDrop) );
            }
        }
    }

    /* Add trace callbacks to the Queue on the left side of
     * the bottleneck link.
     */
    if( param.trace_packets_in_queue ||
        param.trace_bytes_in_queue ||
        param.trace_time_in_queue )
    {
        /* Hook a queue length indicator into the router devices */
        Ptr<NetDevice> nd = routerdevices.Get(0);
        Ptr<PointToPointNetDevice> ptpnd = DynamicCast<PointToPointNetDevice> (nd);
        Ptr<Queue<Packet> > queue = ptpnd->GetQueue ();

        /*
         * This is a useful trace
         */
        if( param.trace_packets_in_queue )
            queue->TraceConnectWithoutContext( "PacketsInQueue",
                                               MakeCallback (&DevicePacketsInQueueTrace) );

        /*
         * This is a useful trace
         */
        if( param.trace_bytes_in_queue )
            queue->TraceConnectWithoutContext( "BytesInQueue",
                                               MakeCallback (&DeviceBytesInQueueTrace) );

        /*
         * This trace does not work
         */
        if( param.trace_time_in_queue )
            queue->TraceConnectWithoutContext( "SojournTime",
                                               MakeCallback (&DeviceTimeInQueueTrace) );
    }

    /* Use the Connect mechanism to add trace callbacks to the TCP stacks.
     * This prints information from the right and left end systems. The number
     * that appears after /NodeList/ determines the node.
     */
    if( param.trace_rtt)
    {
        ns3::Config::Connect( "/NodeList/*/$ns3::TcpL4Protocol/SocketList/0/RTT",
                              ns3::MakeCallback(&RttTracer) );
    }
    if( param.trace_cwnd )
    {
        ns3::Config::Connect( "/NodeList/*/$ns3::TcpL4Protocol/SocketList/0/CongestionWindow",
                              ns3::MakeCallback(&CwndTracer) );
    }
    if( param.trace_ssthresh )
    {
        ns3::Config::Connect( "/NodeList/*/$ns3::TcpL4Protocol/SocketList/0/SlowStartThreshold",
                              ns3::MakeCallback(&SsThreshTracer) );
    }

    /* Some ns3 classes have the member functions EnableAscii or EnablePcap,
     * which print packets to a given stream.
     * These functions can hide quite detailed tracing capabilities.
     * EnableAscii allows writing of human-readable files.
     *             The output starts with a link layer header of some kind,
     *             followed by IP header, following byte transport header.
     * EnablePcap allows writing of PCAP files that can be opened with tools
     *            such as Wireshark.
     */
    if( param.trace_packets )
    {
        AsciiTraceHelper         a;
        Ptr<OutputStreamWrapper> out;

        out = a.CreateFileStream( param.ostream_prefix + "leafnodetrace.tr" );
        pointToPointLeaf.EnableAscii( out, leftrouterdevices );

        out = a.CreateFileStream( param.ostream_prefix + "bottlenecktrace.tr" );
        bottleNeckLink.EnableAscii( out, routerdevices.Get(0) );

        out = a.CreateFileStream( param.ostream_prefix + "internetstack.tr" );
        stack.EnableAsciiIpv4All( out );
        // stack.EnableAsciiIpv4( out3, routerifs );
    }

    /*************************************************************
     * Phase IV
     *
     * Run the application
     *************************************************************/
    sinkApps.Start(ns3::Seconds(0.0));

    if( param.simstop > 0.0 )
    {
        ns3::Simulator::Stop( ns3::Seconds(param.simstop) );
    }

    ns3::Simulator::Run();

    /*************************************************************
     * Phase V
     *
     * Simulation shutdown.
     * Print some final output.
     *************************************************************/
    std::cerr << "Dumbbell Left Bottleneck NodeID " <<
        routers.Get(0)->GetId() << std::endl;

    std::cerr << routers.Get(0)->GetInstanceTypeId().GetName() << std::endl;

    std::cerr << "Dumbbell Right Bottleneck NodeID " <<
        routers.Get(1)->GetId() << std::endl;
    for (uint32_t i = 0; i<number_of_clients; ++i) {
        std::cerr << "Dumbbell Left Leaf " << i << " NodeID " <<
            leftleaves.Get(i)->GetId() << " IP Addr " <<
            leftleafifs.GetAddress(i) << std::endl;
    }
    for (uint32_t i = 0; i<number_of_clients; ++i) {
        std::cerr << "Dumbbell Right Leaf " << i << " NodeID " <<
            rightleaves.Get(i)->GetId() << " IP Addr " << rightleafifs.
            GetAddress(i) << std::endl;
    }

    {
        Ptr<TrafficControlLayer> tc = routers.Get(0)->GetObject<TrafficControlLayer>();
        if( tc != 0 )
        {
            Ptr<QueueDisc> qd = tc->GetRootQueueDiscOnDevice( routerdevices.Get(0) );
            if( qd != 0 )
            {
                QueueDisc::Stats st = qd->GetStats();
                std::cerr << "Total number of packets dropped: " << st.nTotalDroppedPackets << std::endl;
            }
        }   
    }

    ns3::Simulator::Destroy();
    return 0;
}