Merge pull request #34 from madebyhiro/pr/33

HLS jobs

Author: Sjoerd Tieleman

README.markdown

@@ -134,6 +134,9 @@ Parameters (HTTP POST data, should be valid JSON object):
             "size": "160x90",
             "format": "png"
         },
+        "segments_options" : {
+          "segment_time": 10
+        },
         "callback_urls": ["http://example.com/notifications"]
     }
 
@@ -143,6 +146,9 @@ Responses:
 * `400 Bad Request` - Invalid request (format)
 * `503 Service Unavailable` - Transcoder not accepting jobs at the moment (all encoding slots are in use)
 
+
+Required options are `source_file`, `destination_file` and `encoder_options`. Input and output files must be *absolute* paths.
+
 The `callback_urls` array is optional and is a list (array) of HTTP endpoints that should be notified once encoding finishes (due to the job being complete or some error condition). The notification will sent using HTTP PUT to the specified endpoints with the job status. It will also include a custom HTTP header "X-Codem-Notify-Timestamp" that contains the timestamp (in milliseconds) at which the notification was generated and sent. It is best to observe this header to determine the order in which notifications are received at the other end due to network lag or other circumstances that may cause notifications to be received out of order.
 
 The `thumbnail_options` object is optional and contains a set of thumbnails that should be encoded after the transcoding is complete. Thumbnails are captured from the source file for maximum quality. The options for thumbnails include:
@@ -159,7 +165,17 @@ The `thumbnail_options` object is optional and contains a set of thumbnails that
 
 If you specify thumbnails but an error occurs during generation, your job will be marked as failed. If you don't specify a valid `seconds` or `percentages` option thumbnail generation will be skipped but the job can still be completed successfully.
 
-All other options are required (`source_file`, `destination_file` and `encoder_options`). Input and output files must be *absolute* paths.
+The `segments_options` object is optional and contains segment time (duration) in seconds. Segmented videos are used in [HLS](https://en.wikipedia.org/wiki/HTTP_Live_Streaming). These options are applied to the encoded video, thus `encoder_options` are required. Moreover `encoder_options` should prepare video for segmenting, because bitstream
+filter [h264_mp4toannexb](https://www.ffmpeg.org/ffmpeg-bitstream-filters.html#h264_005fmp4toannexb) will be applied to the video. Therefore it is recommended to transcode to an MP4 file before segmenting.
+
+The segmenting command looks like:
+
+    ffmpeg -i /tmp/46ee0a404a4b75d85c09d98a7c6b403579ee9f99.mp4 -codec copy -map 0 \
+      -f segment -vbsf h264_mp4toannexb -flags -global_header -segment_format mpegts \
+      -segment_list /path/to/dest_file_dir/dest_file_name.m3u8 -segment_time 10 \
+      /path/to/dest_file_dir/dest_file_name-%06d.ts
+
+`46ee0a404a4b75d85c09d98a7c6b403579ee9f99.mp4` is a temporary encoded file (generated by Codem). After transcoding and segmenting you end up with the transcoded file, as well as the segments/playlist.
 
 ### Thumbnail-only job
 
@@ -259,6 +275,61 @@ Thumbnail-only job (160x90 in PNG format every 10% of the video).
 
     Output: {"message":"The transcoder accepted your job.","job_id":"d4b1dfebe6860839b2c21b70f35938d870011682"}
 
+
+Segmenting job.
+
+    # curl -d '{"source_file": "/tmp/video.mp4", "destination_file": "/tmp/output/test.mp4", "encoder_options": "-vb 2000k -minrate 2000k -maxrate 2000k -bufsize 2000k -s 1280x720 -acodec aac -strict -2 -ab 192000 -ar 44100 -ac 2 -vcodec libx264 -movflags faststart", "segments_options": {"segment_time": 10} }' http://localhost:8080/jobs
+
+    Output: {"message":"The transcoder accepted your job.","job_id":"7dc3c268783d7f3c737f3a134ccf1d4f15bb8442"}
+
+    Status of finished job:
+
+    # curl http://localhost:8080/jobs/7dc3c268783d7f3c737f3a134ccf1d4f15bb8442
+
+    Output:
+    {
+      "id": "7dc3c268783d7f3c737f3a134ccf1d4f15bb8442",
+      "status": "success",
+      "progress": 1,
+      "duration": 1,
+      "filesize": 783373,
+      "message": "ffmpeg finished succesfully.",
+      "playlist": "/tmp/output/test.m3u8",
+      "segments": [
+        "/tmp/output/test-000000.ts",
+        "/tmp/output/test-000001.ts",
+        "/tmp/output/test-000002.ts",
+        "/tmp/output/test-000003.ts",
+        "/tmp/output/test-000004.ts",
+        "/tmp/output/test-000005.ts"
+      ]
+    }
+
+Segmenting-only job (you are expected to have a valid MP4 file suitable for segmenting as the input).
+
+    # curl -d '{"source_file": "/tmp/video.mp4","destination_file":"/tmp/segments/video.mp4","encoder_options": "", "segments_options": {"segment_time": 10} }' http://localhost:8080/jobs
+
+    Output: {"message":"The transcoder accepted your job.","job_id":"c7599790527c0bb173cc7a0c44411aaca5c1550a"}
+
+    Status of finished job:
+    
+    # curl http://localhost:8080/jobs/c7599790527c0bb173cc7a0c44411aaca5c1550a
+    
+    Output:
+    {
+      "id":"c7599790527c0bb173cc7a0c44411aaca5c1550a",
+      "status":"success",
+      "progress":1,
+      "duration":26,
+      "filesize":6734045,
+      "message":"finished segmenting job.",
+      "playlist":"/tmp/segments/video.m3u8",
+      "segments":[
+        "/tmp/segments/video-000000.ts",
+        "/tmp/segments/video-000001.ts",
+        "/tmp/segments/video-000002.ts"
+      ]
+    }
 ## Issues and support
 
 If you run into any issues while using codem-transcode please use the Github issue tracker to see if it is a known problem
@@ -269,4 +340,4 @@ commercial support or are already receiving commercial support, feel free to con
 
 ## License
 
-Codem-transcode is released under the MIT license, see `LICENSE.txt`.
+Codem-transcode is released under the MIT license, see `LICENSE.txt`.

lib/job-handler.js

@@ -98,14 +98,15 @@ function validateJobRequest(postData, callback) {
   // destination_file:  the output file
   // encoder_options:   the flags for the encoder
   // thumbnail_options: options to generate thumbnails (optional)
+  // segments_options:  options to generate segmented video (optional)
   // callback_urls:     array of callbacks to notify of events (optional)
   var missingFields = [];
   var opts = {};
 
   try {
     var obj = JSON.parse(postData);
     var requiredFields = ['source_file', 'destination_file', 'encoder_options'];
-    var acceptedFields = ['source_file', 'destination_file', 'encoder_options', 'thumbnail_options', 'callback_urls'];
+    var acceptedFields = ['source_file', 'destination_file', 'encoder_options', 'thumbnail_options', 'segments_options', 'callback_urls'];
 
     for (var field in requiredFields) {
       if (typeof(obj[requiredFields[field]]) == "undefined") {
@@ -157,4 +158,4 @@ function hasFreeSlots() {
 function removeItemFromSlot(item) {
   var idx = slots.indexOf(item);
   if(idx!=-1) slots.splice(idx, 1);
-}
+}

lib/job.js

@@ -144,6 +144,8 @@ var Job = JobUtils.getDatabase().define('Job', {
   opts:        { type: Sequelize.TEXT, defaultValue: null },
   thumbnails:  { type: Sequelize.TEXT, defaultValue: null },
   message:     { type: Sequelize.TEXT, defaultValue: null },
+  playlist:    { type: Sequelize.STRING, defaultValue: null },
+  segments:    { type: Sequelize.TEXT, defaultValue: null },
   createdAt:   Sequelize.DATE,
   updatedAt:   Sequelize.DATE
 }, {
@@ -274,35 +276,104 @@ var Job = JobUtils.getDatabase().define('Job', {
       }
     },
     didFinish: function(code) {
-      if (code == 0 && this.parsedOpts()['thumbnail_options']) {
-        this.processThumbnails();
-      } else {
+      if (code != 0) {
         this.finalize(code);
+        return;
+      }
+
+      this.processThumbnails({
+        error:   function(job) { job.finalize(1); },
+        success: function(job) {
+          job.processSegments({
+            error:   function(job) { job.finalize(1); },
+            success: function(job) { job.finalize(0); }
+          })
+        }
+      })
+    },
+
+    processSegments: function(callbacks){
+      if (!this.parsedOpts()['segments_options']) {
+        callbacks.success(this);
+        return;
       }
+
+      logger.log("Processing segments for job " + this.internalId + ".");
+
+      var job              = this;
+      var args             = [];
+      var segmentsOpts     = this.parsedOpts()['segments_options'];
+      var segmentTime      = segmentsOpts['segment_time'];
+      var destinationFile  = this.parsedOpts()['destination_file'];
+      var playlistName     = path.basename(destinationFile, path.extname(destinationFile));
+      var playlistDir      = path.dirname(destinationFile);
+      var playlistPath     = [playlistDir, playlistName].join(path.sep) + '.m3u8';
+      var segmentsFormat   = [playlistDir, playlistName].join(path.sep) + '-%06d.ts'
+      var inputFile        = (this.parsedOpts()['encoder_options'].length > 0) ? job.tmpFile : this.parsedOpts()['source_file']
+      
+      args.push('-i', inputFile,
+                '-codec', 'copy', '-map', '0', '-f', 'segment',
+                '-vbsf', 'h264_mp4toannexb', '-flags', '-global_header',
+                '-segment_format', 'mpegts', '-segment_list', playlistPath,
+                '-segment_time', segmentTime, segmentsFormat);
+
+      child_process.execFile(config['encoder'], args, function(error, stdout, stderr) {
+        if (error) {
+          job.lastMessage = 'Error while generating segments: ' + error.message;
+          callbacks.error(job);
+          return;
+        }
+
+        fs.readdir(playlistDir, function(error, files) {
+          if (error) {
+            job.lastMessage = 'Error while generating segments: ' + error.message;
+            callbacks.error(job);
+            return;
+          }
+
+          job.segments = JSON.stringify(files.filter(
+            function(file){ return file.match(new RegExp(playlistName + "-\\d+\\.ts")) }
+          ).map(
+            function(file){ return path.join(playlistDir, file) }
+          ));
+
+          job.playlist = playlistPath;
+
+          callbacks.success(job);
+        });
+      });
     },
-    processThumbnails: function() {
+
+    processThumbnails: function(callbacks) {
+      if (!this.parsedOpts()['thumbnail_options']) {
+        callbacks.success(this);
+        return;
+      }
+
       logger.log("Processing thumbnails for job " + this.internalId + ".");
       var thumbOpts = this.parsedOpts()['thumbnail_options'];
       var range = JobUtils.generateRangeFromThumbOpts(thumbOpts, this.duration);
-      
-      if (range) {
-        var job = this;
-        async.parallel(
-          range.map(job.execThumbJob.bind(job)),
-          function(err, results) {
-            if (err) {
-              job.lastMessage = err.message;
-              job.finalize(1);
-            } else {
-              job.finalize(0, results);
-            }
-          }
-        );
-      } else {
+
+      if (!range) {
         // no valid range
         logger.log("No valid thumbnails to process for job " + this.internalId + ". Skipping...");
-        this.finalize(0);
+        callbacks.success(this);
+        return;
       }
+
+      var job = this;
+      async.parallel(
+        range.map(job.execThumbJob.bind(job)),
+        function(err, results) {
+          if (err) {
+            job.lastMessage = err.message;
+            callbacks.error(job);
+          } else {
+            job.thumbnails = JSON.stringify(results);
+            callbacks.success(job);
+          }
+        }
+      );
     },
     execThumbJob: function(offset) {
       var job = this;
@@ -331,43 +402,48 @@ var Job = JobUtils.getDatabase().define('Job', {
         });
       }
     },
-    finalize: function(code, thumbnails) {
+    finalize: function(code) {
       var job = this;
-      if (thumbnails) job.thumbnails = JSON.stringify(thumbnails);
-      
-      if (code == 0) {
-        if (job.tmpFile) {
-          fs.rename(job.tmpFile, job.parsedOpts()['destination_file'], function (err) {
-            if (err) {
-              if ( (err.message).match(/EXDEV/) ) {
-                /*
-                  EXDEV fix, since util.pump is deprecated, using stream.pipe
-                  example from http://stackoverflow.com/questions/11293857/fastest-way-to-copy-file-in-node-js
-                */
-                try {
-                  logger.log('ffmpeg finished successfully, trying to copy across partitions');
-                  fs.createReadStream(job.tmpFile).pipe(fs.createWriteStream(job.parsedOpts()['destination_file']));
-                  job.exitHandler(code, 'ffmpeg finished succesfully.');
-                } catch (err) {
-                  logger.log(err);
-                  job.exitHandler(-1, 'ffmpeg finished succesfully, but unable to move file to different partition (' + job.parsedOpts()['destination_file'] + ').');
-                }
 
-              } else {
-                logger.log(err);
-                job.exitHandler(-1, 'ffmpeg finished succesfully, but unable to move file to destination (' + job.parsedOpts()['destination_file'] + ').');
-              }
-            } else {
+      if (code != 0) {
+        job.exitHandler(code, "ffmpeg finished with an error: '" + job.lastMessage + "' (" + code + ").");
+        return;
+      }
+
+      if (!job.tmpFile) {
+        // No tmpFile, hence no transcoding, only thumbnails or segmenting
+        if (job.parsedOpts()['thumbnail_options']) {
+          job.exitHandler(code, 'finished thumbnail job.');
+        } else {
+          job.exitHandler(code, 'finished segmenting job.');
+        }
+        return;
+      }
+
+      fs.rename(job.tmpFile, job.parsedOpts()['destination_file'], function (err) {
+        if (err) {
+          if ( (err.message).match(/EXDEV/) ) {
+            /*
+              EXDEV fix, since util.pump is deprecated, using stream.pipe
+              example from http://stackoverflow.com/questions/11293857/fastest-way-to-copy-file-in-node-js
+            */
+            try {
+              logger.log('ffmpeg finished successfully, trying to copy across partitions');
+              fs.createReadStream(job.tmpFile).pipe(fs.createWriteStream(job.parsedOpts()['destination_file']));
               job.exitHandler(code, 'ffmpeg finished succesfully.');
+            } catch (err) {
+              logger.log(err);
+              job.exitHandler(-1, 'ffmpeg finished succesfully, but unable to move file to different partition (' + job.parsedOpts()['destination_file'] + ').');
             }
-          });          
+
+          } else {
+            logger.log(err);
+            job.exitHandler(-1, 'ffmpeg finished succesfully, but unable to move file to destination (' + job.parsedOpts()['destination_file'] + ').');
+          }
         } else {
-          // No tmpFile, hence no transcoding, only thumbnails
-          job.exitHandler(code, 'finished thumbnail job.');
+          job.exitHandler(code, 'ffmpeg finished succesfully.');
         }
-      } else {
-        job.exitHandler(code, "ffmpeg finished with an error: '" + job.lastMessage + "' (" + code + ").")
-      }
+      });
     },
     toJSON: function() {
       var obj = {
@@ -377,13 +453,17 @@ var Job = JobUtils.getDatabase().define('Job', {
         'duration': this.duration,
         'filesize': this.filesize,
         'message': this.message,
-        
       };
 
       if (this.thumbnails) {
         obj['thumbnails'] = JSON.parse(this.thumbnails);
       }
-      
+
+      if (this.playlist) { obj['playlist'] = this.playlist; }
+      if (this.segments) {
+        obj['segments'] = JSON.parse(this.segments);
+      }
+
       return obj;
     },
     progressHandler: function(data) {
@@ -448,4 +528,4 @@ var Job = JobUtils.getDatabase().define('Job', {
   }
 });
 
-module.exports = Job;
+module.exports = Job;

migrations/20140822085031-add-playlist-and-segments-to-jobs.js

@@ -0,0 +1,12 @@
+module.exports = {
+  up: function(migration, DataTypes, done) {
+    // add altering commands here
+    migration.addColumn('Jobs', 'playlist', { type: DataTypes.STRING, defaultValue: null }).complete(done);
+    migration.addColumn('Jobs', 'segments', { type: DataTypes.TEXT, defaultValue: null }).complete(done);
+  },
+  down: function(migration, DataTypes, done) {
+    // add reverting commands here
+    migration.removeColumn('Jobs', 'playlist').complete(done);
+    migration.removeColumn('Jobs', 'segments').complete(done);
+  }
+}