How to process images real-time from the iOS camera

Learn how to process real-time camera feed images for computer vision features

Vector graphics sourced from https://www.vecteezy.com

Getting started

1. Create a new app

2. Display the camera feed

import AVFoundation
private let captureSession = AVCaptureSession()
private func addCameraInput() {
let device = AVCaptureDevice.default(for: .video)!
let cameraInput = try! AVCaptureDeviceInput(device: device)
self.captureSession.addInput(cameraInput)
}
self.addCameraInput()
Added camera input to the capture session coordinator
private lazy var previewLayer: AVCaptureVideoPreviewLayer = {
let preview = AVCaptureVideoPreviewLayer(session: self.captureSession)
preview.videoGravity = .resizeAspect
return preview
}()
private func addPreviewLayer() {
self.view.layer.addSublayer(self.previewLayer)
}
self.addPreviewLayer()
self.captureSession.startRunning()
override func viewDidLayoutSubviews() {
super.viewDidLayoutSubviews()
self.previewLayer.frame = self.view.bounds
}
Added camera preview layer to screen

3. Processing the camera feed

private let videoOutput = AVCaptureVideoDataOutput()
private func addVideoOutput() {
self.videoOutput.videoSettings = [(kCVPixelBufferPixelFormatTypeKey as NSString) : NSNumber(value: kCVPixelFormatType_32BGRA)] as [String : Any]
self
.videoOutput.setSampleBufferDelegate(self, queue: DispatchQueue(label: "my.image.handling.queue"))
self.captureSession.addOutput(self.videoOutput)
}
Xcode error
class ViewController: UIViewController, AVCaptureVideoDataOutputSampleBufferDelegate {
func captureOutput(_ output: AVCaptureOutput,
didOutput sampleBuffer: CMSampleBuffer,
from connection: AVCaptureConnection) {
guard let frame = CMSampleBufferGetImageBuffer(sampleBuffer) else {
debugPrint("unable to get image from sample buffer")
return
}
print("did receive image frame")
// process image here
}
self.addVideoOutput()
console logging every time we receive a live frame

Summary

Final Notes

Senior iOS Engineer @ Onfido. Writing weekly blogs on iOS and programming. Follow me to stay tuned!

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store