frame.h

Go to the documentation of this file.

#ifndef frame_HEADER_GUARD
#define frame_HEADER_GUARD
 
/*
 * frame.h : Frame classes
 * 
 * Copyright 2017-2020 Valkka Security Ltd. and Sampsa Riikonen
 * 
 * Authors: Sampsa Riikonen <sampsa.riikonen@iki.fi>
 * 
 * This file is part of the Valkka library.
 * 
 * Valkka is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>
 *
 */
 
#include "common.h"
#include "codec.h"
#include "metadata.h"
#include "threadsignal.h"
#include "constant.h"
#include "avdep.h"
#include "opengl.h"
#include "tools.h"
#include "macro.h"
#include "rawrite.h"
 
enum class FrameClass
{
    none, 
 
    basic, 
 
    avpkt, 
 
    avmedia,  
    avbitmap, 
    // avbitmap_np,  ///< child of avmedia: video, non-planar
 
    avaudio, 
 
    avrgb, 
 
    yuv, 
 
    rgb, 
 
    setup,  
    signal, 
 
    marker, 
 
    mux, 
 
    First = none,
    Last = mux
};
 
enum class TimeCorrectionType // <pyapi>
{          // <pyapi>
    none,  // <pyapi>
    smart, // <pyapi>
    dummy  // <pyapi>
};         // <pyapi>
 
class Frame
{
 
public:
    Frame();          
    virtual ~Frame(); 
    frame_essentials(FrameClass::none, Frame);
    frame_clone(FrameClass::none, Frame);
    /*Frame(const Frame &f); ///< Default copy ctor
  
  
public: // frame essentials : must be defined for each frame subclass
  virtual FrameClass getFrameClass(); ///< Returns the subclass frame type.  See Frame::frameclass
  virtual void copyFrom(Frame *f);    ///< Copies data to this frame from a frame of the same type (also metadata)
  */
 
public:                                                  // redefined virtual
    virtual void print(std::ostream &os) const;          
    virtual std::string dumpPayload();                   
    virtual void dumpPayloadToFile(std::ofstream &fout); 
    virtual void updateAux();                        
    virtual void update();                           
    virtual void reset();                                
    virtual bool isSeekable();                           
 
public:
    void copyMetaFrom(Frame *f); 
 
protected:
    FrameClass frameclass; 
 
public:                   // public metadata
    SlotNumber n_slot;    
    int subsession_index; 
    long int mstimestamp; 
};
 
inline std::ostream &operator<<(std::ostream &os, const Frame &f)
{
    // https://stackoverflow.com/questions/4571611/making-operator-virtual
    f.print(os);
    return os;
}
 
class BasicFrame : public Frame
{
 
public:
    BasicFrame();          
    virtual ~BasicFrame(); 
    frame_essentials(FrameClass::basic, BasicFrame);
    frame_clone(FrameClass::basic, BasicFrame);
    /*BasicFrame(const BasicFrame &f); ///< Default copy ctor
  
public: // frame essentials
  virtual FrameClass getFrameClass();         ///< Returns the subclass frame type.  See Frame::frameclass
  virtual void copyFrom(Frame *f);            ///< Copies data to this frame from a frame of the same type
  */
 
public:                                         // redefined virtual
    virtual void print(std::ostream &os) const; 
    virtual std::string dumpPayload();
    virtual void dumpPayloadToFile(std::ofstream &fout);
    virtual void reset();      
    virtual bool isSeekable(); 
 
public:                                // payload handling
    void reserve(std::size_t n_bytes); 
    void resize(std::size_t n_bytes);  
 
public:                           // frame variables
    std::vector<uint8_t> payload; 
    AVMediaType media_type;       
    AVCodecID codec_id;           
 
public:                 // codec-dependent parameters
    H264Pars h264_pars; 
 
public:                  // codec-dependent functions
    void fillPars();     
    void fillH264Pars(); 
 
public:
    void fillAVPacket(AVPacket *avpkt);                                                                    
    void copyFromAVPacket(AVPacket *avpkt);                                                                
    void filterFromAVPacket(AVPacket *avpkt, AVCodecContext *codec_ctx, AVBitStreamFilterContext *filter); 
 
public:                                                  // frame serialization
    std::size_t calcSize();                              
    bool dump(IdNumber device_id, RaWriter &raw_writer); 
    IdNumber read(RawReader &raw_reader);                
};
 
class MuxFrame : public Frame {
 
public:
    MuxFrame(); 
    virtual ~MuxFrame(); 
    frame_essentials(FrameClass::mux, MuxFrame);
    frame_clone(FrameClass::mux, MuxFrame);
        
public: // redefined virtual
    virtual void print(std::ostream& os) const;             
    virtual std::string dumpPayload();                      
    virtual void dumpPayloadToFile(std::ofstream& fout);    
    virtual void reset();                                   
    //virtual bool isSeekable();                              ///< Can we seek to this frame? 
 
/*
public:
    virtual bool isInit();  ///< for frag-MP4: ftyp, moov
    virtual bool isMeta();  ///< for frag-MP4: moof
*/
 
public:                                // payload handling
    void reserve(std::size_t n_bytes); 
    void resize(std::size_t n_bytes);  
 
public:
    std::vector<uint8_t> payload; 
    AVMediaType media_type;       
    AVCodecID codec_id;           
 
public:
    std::vector<uint8_t> meta_blob; 
    MuxMetaType          meta_type; 
};
 
 
enum class SetupFrameType
{
    none,
    stream_init,
    stream_state
};
 
class SetupFrame : public Frame
{
 
public:
    SetupFrame();          
    virtual ~SetupFrame(); 
    frame_essentials(FrameClass::setup, SetupFrame);
    frame_clone(FrameClass::setup, SetupFrame);
    /*
  SetupFrame(const SetupFrame &f); ///< Default copy ctor
  
public: // frame essentials
  virtual FrameClass getFrameClass();         ///< Returns the subclass frame type.  See Frame::frameclass
  virtual void copyFrom(Frame *f);            ///< Copies data to this frame from a frame of the same type
  */
 
public:                                         // redefined virtual
    virtual void print(std::ostream &os) const; 
    virtual void reset();                       
 
public:                      // managed objects
    SetupFrameType sub_type; 
 
    AVMediaType media_type; 
    AVCodecID codec_id;     
 
    AbstractFileState stream_state; 
};
 
class AVMediaFrame : public Frame
{
 
public:
    AVMediaFrame();          
    virtual ~AVMediaFrame(); 
    //frame_essentials(FrameClass::avmedia,AVMediaFrame); // now this is a virtual class ..
    //frame_clone(FrameClass::avmedia,AVMediaFrame);
    /*AVMediaFrame(const AVMediaFrame &f); ///< Default copy ctor
    
    public: // frame essentials
    virtual FrameClass getFrameClass(); ///< Returns the subclass frame type.  See Frame::frameclass
    virtual void copyFrom(Frame *f);    ///< Copies data to this frame from a frame of the same type
    */
 
/*
public:
    virtual void updateAux() = 0; ///< update any helper objects
    virtual void update() = 0;
*/
 
public: // redefined virtual
    virtual std::string dumpPayload();
    virtual void print(std::ostream &os) const; 
    virtual void reset();                       
 
public:                     // helper objects : values should correspond to member av_frame
    AVMediaType media_type; 
    AVCodecID codec_id;     
 
public:                // managed objects
    AVFrame *av_frame; 
};
 
class AVBitmapFrame : public AVMediaFrame
{
 
public:
    AVBitmapFrame();          
    virtual ~AVBitmapFrame(); 
    frame_essentials(FrameClass::avbitmap, AVBitmapFrame);
    frame_clone(FrameClass::avbitmap, AVBitmapFrame); // TODO: think about this!
                                                      /*
  AVBitmapFrame(const AVBitmapFrame &f); ///< Default copy ctor
  
public: // frame essentials
  virtual FrameClass getFrameClass(); ///< Returns the subclass frame type.  See Frame::frameclass
  virtual void copyFrom(Frame *f);    ///< Copies data to this frame from a frame of the same type
  */
 
public: // redefined virtual
    virtual std::string dumpPayload();
    virtual void print(std::ostream &os) const; 
    virtual void reset();                       
    virtual void copyPayloadFrom(AVBitmapFrame *frame);
    virtual void updateAux();                   
    virtual void update();
    virtual void reserve(int width, int height); 
 
public:                            // helper objects
    AVPixelFormat av_pixel_format; 
    BitmapPars bmpars;             
    uint8_t *y_payload;            
    uint8_t *u_payload;            
    uint8_t *v_payload;            
};
 
class AVRGBFrame : public AVBitmapFrame
{
 
public:
    AVRGBFrame();
    virtual ~AVRGBFrame();
    frame_essentials(FrameClass::avrgb, AVRGBFrame);
    frame_clone(FrameClass::avrgb, AVRGBFrame); // TODO: think about this!
 
public:
    virtual void reserve(int width, int height); 
 
public: // redefined virtual
    virtual std::string dumpPayload();
    virtual void print(std::ostream &os) const; 
};
 
/*
class AVAudioFrame : public AVMediaFrame {
  
public:
  AVAudioFrame();
  virtual ~AVAudioFrame();
  
public: // frame essentials
  virtual FrameClass getFrameClass(); ///< Returns the subclass frame type.  See Frame::frameclass
  virtual void copyFrom(Frame *f);    ///< Copies data to this frame from a frame of the same type
  virtual std::string dumpPayload();
  virtual void print(std::ostream& os) const; ///< How to print this frame to output stream
  
public:
  virtual void getParametersDecoder(const AVCodecContext *ctx); ///< Extract sample rate
  
public: // ffmpeg media parameters
  AVSampleFormat av_sample_fmt;
};
*/
 
// Decoder gives AVBitmapFrame => OpenGLThread internal FrameFilter => OpenGLThread FrameFifo::writeCopy(Frame *f) => .. returns once a copy has been made
 
class YUVFrame : public Frame
{
 
public:
    YUVFrame(BitmapPars bmpars); 
    virtual ~YUVFrame();         
    frame_essentials(FrameClass::yuv, YUVFrame);
    // TODO: frame_clone
    /*
  YUVFrame(const YUVFrame &f); ///< Default copy ctor
  
public: // frame essentials
  virtual FrameClass getFrameClass(); ///< Returns the subclass frame type.  See Frame::frameclass
  virtual void copyFrom(Frame *f);    ///< Copies data to this frame from a frame of the same type
 
  */
 
public:                       // variables filled at constructor time
    BitmapPars bmpars;        // the maximum, pre-reserved size
    BitmapPars source_bmpars; // the actual size of the image
 
private:
    void reserve(); 
    void release(); 
 
public:             // variables used/filled by reserve
    GLuint y_index; 
    GLuint u_index; 
    GLuint v_index; 
 
private:
    GLubyte *y_payload; 
    GLubyte *u_payload; 
    GLubyte *v_payload; 
 
public: // redefined virtual
    virtual std::string dumpPayload();
    virtual void print(std::ostream &os) const; 
    virtual void reset();
 
public:
    void fromAVBitmapFrame(AVBitmapFrame *f); 
};
 
class RGBFrame : public Frame
{
 
public:
    RGBFrame(int max_width, int max_height);
    virtual ~RGBFrame();
    frame_essentials(FrameClass::rgb, RGBFrame);
 
private:
    std::vector<uint8_t> payload; 
    int max_width, max_height;
    int width, height;
 
public: // redefined virtual
    virtual std::string dumpPayload();
    virtual void print(std::ostream &os) const; 
    virtual void reset();
 
public:
    void fromAVRGBFrame(AVRGBFrame *f); 
};
 
typedef std::vector<Frame *> Reservoir;
typedef std::deque<Frame *> Stack;
typedef std::deque<Frame *> Fifo;
typedef std::deque<Frame *> Cache;
 
typedef std::vector<YUVFrame *> YUVReservoir;
typedef std::deque<YUVFrame *> YUVStack;
typedef std::vector<RGBFrame *> RGBReservoir;
typedef std::deque<RGBFrame *> RGBStack;
 
 
class SignalFrame : public Frame
{
 
public:
    SignalFrame();          
    virtual ~SignalFrame(); 
    frame_essentials(FrameClass::signal, SignalFrame);
    frame_clone(FrameClass::signal, SignalFrame);
 
public:
    unsigned signaltype; 
 
public:
    OpenGLSignalContext opengl_signal_ctx;                 
    AVSignalContext av_signal_ctx;                         
    ValkkaFSWriterSignalContext valkkafswriter_signal_ctx; 
    ValkkaFSReaderSignalContext valkkafsreader_signal_ctx; 
    // TODO: those ctxes should be written with the new technique:
    // this generalizes & is copyable
    std::vector<uint8_t> signal_ctx_buf;                
    /* example of the (revised) signal handling
    if (signalframe->signaltype==SignalType::offline) {
        OfflineSignalContext ctx = OfflineSignalContext();
        get_signal_context(signalframe, ctx);
        # that's a macro: take a look at macro.h
    }
    */
 
public:
    virtual void reset();
 
};
 
class MarkerFrame : public Frame
{
 
public:
    MarkerFrame();          
    virtual ~MarkerFrame(); 
    frame_essentials(FrameClass::marker, MarkerFrame);
    frame_clone(FrameClass::marker, MarkerFrame);
 
public: // redefined virtual
    // virtual std::string dumpPayload();
    virtual void print(std::ostream &os) const; 
    virtual void reset();
 
public:
    bool fs_start, fs_end; 
    bool tm_start, tm_end; 
};
 
#endif