2020-1-capstone-design2 / 2014104089

Go to a project
Toggle navigation Toggle navigation pinning
Hoya-kim
a6d626b1 1 parent f0e1919f

[ADD] 최종보고서, 면담보고서 및 소스코드 제출

Expand all
Inline Side-by-side
Showing 31 changed files with 722 additions and 0 deletions
No preview for this file type
No preview for this file type
This diff is collapsed. Click to expand it.
<?xml version="1.0" encoding="UTF-8"?>
<Workspace
version = "1.0">
<FileRef
location = "self:extract_facial_expression.xcodeproj">
</FileRef>
</Workspace>
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>IDEDidComputeMac32BitWarning</key>
<true/>
</dict>
</plist>
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>SchemeUserState</key>
<dict>
<key>extract_facial_expression.xcscheme_^#shared#^_</key>
<dict>
<key>orderHint</key>
<integer>0</integer>
</dict>
</dict>
</dict>
</plist>
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>SchemeUserState</key>
<dict>
<key>extract_facial_expression.xcscheme_^#shared#^_</key>
<dict>
<key>orderHint</key>
<integer>0</integer>
</dict>
</dict>
</dict>
</plist>
//
// AppDelegate.swift
// extract_facial_expression
//
// Created by Jerry kim on 2020/06/11.
// Copyright © 2020 hoya. All rights reserved.
//
import UIKit
@UIApplicationMain
class AppDelegate: UIResponder, UIApplicationDelegate {
func application(_ application: UIApplication, didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]?) -> Bool {
// Override point for customization after application launch.
return true
}
// MARK: UISceneSession Lifecycle
func application(_ application: UIApplication, configurationForConnecting connectingSceneSession: UISceneSession, options: UIScene.ConnectionOptions) -> UISceneConfiguration {
// Called when a new scene session is being created.
// Use this method to select a configuration to create the new scene with.
return UISceneConfiguration(name: "Default Configuration", sessionRole: connectingSceneSession.role)
}
func application(_ application: UIApplication, didDiscardSceneSessions sceneSessions: Set<UISceneSession>) {
// Called when the user discards a scene session.
// If any sessions were discarded while the application was not running, this will be called shortly after application:didFinishLaunchingWithOptions.
// Use this method to release any resources that were specific to the discarded scenes, as they will not return.
}
}
{
"images" : [
{
"idiom" : "iphone",
"scale" : "2x",
"size" : "20x20"
},
{
"idiom" : "iphone",
"scale" : "3x",
"size" : "20x20"
},
{
"idiom" : "iphone",
"scale" : "2x",
"size" : "29x29"
},
{
"idiom" : "iphone",
"scale" : "3x",
"size" : "29x29"
},
{
"idiom" : "iphone",
"scale" : "2x",
"size" : "40x40"
},
{
"idiom" : "iphone",
"scale" : "3x",
"size" : "40x40"
},
{
"idiom" : "iphone",
"scale" : "2x",
"size" : "60x60"
},
{
"idiom" : "iphone",
"scale" : "3x",
"size" : "60x60"
},
{
"idiom" : "ipad",
"scale" : "1x",
"size" : "20x20"
},
{
"idiom" : "ipad",
"scale" : "2x",
"size" : "20x20"
},
{
"idiom" : "ipad",
"scale" : "1x",
"size" : "29x29"
},
{
"idiom" : "ipad",
"scale" : "2x",
"size" : "29x29"
},
{
"idiom" : "ipad",
"scale" : "1x",
"size" : "40x40"
},
{
"idiom" : "ipad",
"scale" : "2x",
"size" : "40x40"
},
{
"idiom" : "ipad",
"scale" : "1x",
"size" : "76x76"
},
{
"idiom" : "ipad",
"scale" : "2x",
"size" : "76x76"
},
{
"idiom" : "ipad",
"scale" : "2x",
"size" : "83.5x83.5"
},
{
"idiom" : "ios-marketing",
"scale" : "1x",
"size" : "1024x1024"
}
],
"info" : {
"author" : "xcode",
"version" : 1
}
}
{
"info" : {
"author" : "xcode",
"version" : 1
}
}
{
"images" : [
{
"filename" : "Extract
Facial Expression
on playing video (2).png",
"idiom" : "universal",
"scale" : "1x"
},
{
"idiom" : "universal",
"scale" : "2x"
},
{
"idiom" : "universal",
"scale" : "3x"
}
],
"info" : {
"author" : "xcode",
"version" : 1
}
}
{
"images" : [
{
"filename" : "backward.png",
"idiom" : "universal",
"scale" : "1x"
},
{
"idiom" : "universal",
"scale" : "2x"
},
{
"idiom" : "universal",
"scale" : "3x"
}
],
"info" : {
"author" : "xcode",
"version" : 1
}
}
{
"images" : [
{
"filename" : "forward.png",
"idiom" : "universal",
"scale" : "1x"
},
{
"idiom" : "universal",
"scale" : "2x"
},
{
"idiom" : "universal",
"scale" : "3x"
}
],
"info" : {
"author" : "xcode",
"version" : 1
}
}
{
"images" : [
{
"filename" : "pause.png",
"idiom" : "universal",
"scale" : "1x"
},
{
"idiom" : "universal",
"scale" : "2x"
},
{
"idiom" : "universal",
"scale" : "3x"
}
],
"info" : {
"author" : "xcode",
"version" : 1
}
}
{
"images" : [
{
"filename" : "play.png",
"idiom" : "universal",
"scale" : "1x"
},
{
"idiom" : "universal",
"scale" : "2x"
},
{
"idiom" : "universal",
"scale" : "3x"
}
],
"info" : {
"author" : "xcode",
"version" : 1
}
}
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="13122.16" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" launchScreen="YES" useTraitCollections="YES" useSafeAreas="YES" colorMatched="YES" initialViewController="01J-lp-oVM">
<dependencies>
<plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="13104.12"/>
<capability name="Safe area layout guides" minToolsVersion="9.0"/>
<capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
</dependencies>
<scenes>
<!--View Controller-->
<scene sceneID="EHf-IW-A2E">
<objects>
<viewController id="01J-lp-oVM" sceneMemberID="viewController">
<view key="view" contentMode="scaleToFill" id="Ze5-6b-2t3">
<rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
<autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
<color key="backgroundColor" xcode11CocoaTouchSystemColor="systemBackgroundColor" cocoaTouchSystemColor="whiteColor"/>
<viewLayoutGuide key="safeArea" id="6Tk-OE-BBY"/>
</view>
</viewController>
<placeholder placeholderIdentifier="IBFirstResponder" id="iYj-Kq-Ea1" userLabel="First Responder" sceneMemberID="firstResponder"/>
</objects>
<point key="canvasLocation" x="53" y="375"/>
</scene>
</scenes>
</document>
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>CFBundleDevelopmentRegion</key>
<string>$(DEVELOPMENT_LANGUAGE)</string>
<key>CFBundleExecutable</key>
<string>$(EXECUTABLE_NAME)</string>
<key>CFBundleIdentifier</key>
<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
<key>CFBundleInfoDictionaryVersion</key>
<string>6.0</string>
<key>CFBundleName</key>
<string>$(PRODUCT_NAME)</string>
<key>CFBundlePackageType</key>
<string>$(PRODUCT_BUNDLE_PACKAGE_TYPE)</string>
<key>CFBundleShortVersionString</key>
<string>1.0</string>
<key>CFBundleVersion</key>
<string>1</string>
<key>LSRequiresIPhoneOS</key>
<true/>
<key>UIApplicationSceneManifest</key>
<dict>
<key>UIApplicationSupportsMultipleScenes</key>
<false/>
<key>UISceneConfigurations</key>
<dict>
<key>UIWindowSceneSessionRoleApplication</key>
<array>
<dict>
<key>UISceneConfigurationName</key>
<string>Default Configuration</string>
<key>UISceneDelegateClassName</key>
<string>$(PRODUCT_MODULE_NAME).SceneDelegate</string>
<key>UISceneStoryboardFile</key>
<string>Main</string>
</dict>
</array>
</dict>
</dict>
<key>UILaunchStoryboardName</key>
<string>LaunchScreen</string>
<key>UIMainStoryboardFile</key>
<string>Main</string>
<key>NSCameraUsageDescription</key>
<string></string>
<key>UIRequiredDeviceCapabilities</key>
<array>
<string>armv7</string>
<string>arkit</string>
</array>
<key>UISupportedInterfaceOrientations</key>
<array>
<string>UIInterfaceOrientationPortrait</string>
<string>UIInterfaceOrientationLandscapeLeft</string>
<string>UIInterfaceOrientationLandscapeRight</string>
</array>
<key>UISupportedInterfaceOrientations~ipad</key>
<array>
<string>UIInterfaceOrientationPortrait</string>
<string>UIInterfaceOrientationPortraitUpsideDown</string>
<string>UIInterfaceOrientationLandscapeLeft</string>
<string>UIInterfaceOrientationLandscapeRight</string>
</array>
</dict>
</plist>
//
// SceneDelegate.swift
// extract_facial_expression
//
// Created by Jerry kim on 2020/06/11.
// Copyright © 2020 hoya. All rights reserved.
//
import UIKit
class SceneDelegate: UIResponder, UIWindowSceneDelegate {
var window: UIWindow?
func scene(_ scene: UIScene, willConnectTo session: UISceneSession, options connectionOptions: UIScene.ConnectionOptions) {
// Use this method to optionally configure and attach the UIWindow `window` to the provided UIWindowScene `scene`.
// If using a storyboard, the `window` property will automatically be initialized and attached to the scene.
// This delegate does not imply the connecting scene or session are new (see `application:configurationForConnectingSceneSession` instead).
guard let _ = (scene as? UIWindowScene) else { return }
}
func sceneDidDisconnect(_ scene: UIScene) {
// Called as the scene is being released by the system.
// This occurs shortly after the scene enters the background, or when its session is discarded.
// Release any resources associated with this scene that can be re-created the next time the scene connects.
// The scene may re-connect later, as its session was not neccessarily discarded (see `application:didDiscardSceneSessions` instead).
}
func sceneDidBecomeActive(_ scene: UIScene) {
// Called when the scene has moved from an inactive state to an active state.
// Use this method to restart any tasks that were paused (or not yet started) when the scene was inactive.
}
func sceneWillResignActive(_ scene: UIScene) {
// Called when the scene will move from an active state to an inactive state.
// This may occur due to temporary interruptions (ex. an incoming phone call).
}
func sceneWillEnterForeground(_ scene: UIScene) {
// Called as the scene transitions from the background to the foreground.
// Use this method to undo the changes made on entering the background.
}
func sceneDidEnterBackground(_ scene: UIScene) {
// Called as the scene transitions from the foreground to the background.
// Use this method to save data, release shared resources, and store enough scene-specific state information
// to restore the scene back to its current state.
}
}
//
// VideoViewController.swift
// extract_facial_expression
//
// Created by Jerry kim on 2020/06/11.
// Copyright © 2020 hoya. All rights reserved.
//
import UIKit
import AVFoundation
import ARKit
class VideoViewController: UIViewController, ARSCNViewDelegate {
let videoPlayerView = UIView()
var player: AVPlayer?
public struct emotions {
let expression: String
let count: Int
}
@IBOutlet var sceneView: ARSCNView!
@IBOutlet weak var expressionLabel: UILabel!
@IBOutlet weak var expression1: UILabel! // smile
@IBOutlet weak var expression2: UILabel! // dumbfounded
@IBOutlet weak var expression3: UILabel! // suprise
@IBOutlet weak var expression4: UILabel! // nyah
@IBOutlet weak var expression5: UILabel! // awful
@IBOutlet weak var expression6: UILabel! // eye smile
@IBOutlet weak var resultLabel: UILabel! // result label
var faceExpression = ""
var count1 = 0 // smile
var count2 = 0 // dumbfounded
var count3 = 0 // suprise
var count4 = 0 // nyah
var count5 = 0 // awful
var count6 = 0 // eye smile
// Sample Videos
let videos = ["http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4",
"http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ElephantsDream.mp4",
"http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerBlazes.mp4",
"http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerEscapes.mp4",
"http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerFun.mp4",
"http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerJoyrides.mp4",
"http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerMeltdowns.mp4",
"http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/SubaruOutbackOnStreetAndDirt.mp4",
"http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/TearsOfSteel.mp4"]
override func viewDidLoad() {
super.viewDidLoad()
videoPlayerView.backgroundColor = UIColor.white
videoPlayerView.translatesAutoresizingMaskIntoConstraints = false
// Add constraints, pinnning each of the four sides
// let topConstraint = NSLayoutConstraint(item: videoPlayerView, attribute: .top, relatedBy: .equal, toItem: view, attribute: .top, multiplier: 1, constant: 0)
// let bottomConstraint = NSLayoutConstraint(item: videoPlayerView, attribute: .bottom, relatedBy: .equal, toItem: view, attribute: .bottom, multiplier: 1, constant: 0)
// let leadingConstraint = NSLayoutConstraint(item: videoPlayerView, attribute: .leading, relatedBy: .equal, toItem: view, attribute: .leading, multiplier: 1, constant: 0)
// let trailingConstraint = NSLayoutConstraint(item: videoPlayerView, attribute: .trailing, relatedBy: .equal, toItem: view, attribute: .trailing, multiplier: 1, constant: 0)
view.addSubview(videoPlayerView)
videoPlayerView.leadingAnchor.constraint(equalTo: view.leadingAnchor).isActive = true
videoPlayerView.trailingAnchor.constraint(equalTo: view.trailingAnchor).isActive = true
videoPlayerView.heightAnchor.constraint(equalTo: view.heightAnchor, multiplier: 0.3).isActive = true
videoPlayerView.topAnchor.constraint(equalTo: view.safeAreaLayoutGuide.topAnchor).isActive = true
// view.addConstraints([topConstraint, bottomConstraint, leadingConstraint, trailingConstraint])
view.sendSubviewToBack(videoPlayerView)
// for ARSCNView
guard ARFaceTrackingConfiguration.isSupported else {
fatalError("Face tracking not available on this on this device model!")
}
sceneView.delegate = self
sceneView.showsStatistics = true
// sceneView.translatesAutoresizingMaskIntoConstraints = false
// sceneView.leadingAnchor.constraint(equalTo: view.leadingAnchor).isActive = true
// sceneView.topAnchor.constraint(equalToSystemSpacingBelow: videoPlayerView.bottomAnchor, multiplier: 0.3).isActive = true
// sceneView.heightAnchor.constraint(equalTo: view.heightAnchor, multiplier: 0.3).isActive = true
}
override var supportedInterfaceOrientations: UIInterfaceOrientationMask {
// for view video, left landscape
return .landscapeLeft
}
func setupVideoPlayer() { // Play random video
let randNum = arc4random_uniform(9)
guard let url = URL(string: self.videos[Int(randNum)]) else {
return
}
player = AVPlayer(url: url)
let playerLayer = AVPlayerLayer(player: player)
playerLayer.frame = videoPlayerView.bounds
videoPlayerView.layer.addSublayer(playerLayer)
player?.play()
}
override func viewWillAppear(_ animated: Bool) {
super.viewWillAppear(animated)
// Create a session configuration
let configuration = ARFaceTrackingConfiguration()
// Run the view's session
sceneView.session.run(configuration)
}
override func viewWillDisappear(_ animated: Bool) {
super.viewWillDisappear(animated)
// Pause the view's session
sceneView.session.pause()
}
override func viewDidAppear(_ animated: Bool) {
super.viewDidAppear(animated)
setupVideoPlayer()
}
func renderer(_ renderer: SCNSceneRenderer, nodeFor anchor: ARAnchor) -> SCNNode? {
let faceMesh = ARSCNFaceGeometry(device: sceneView.device!)
let node = SCNNode(geometry: faceMesh)
node.geometry?.firstMaterial?.fillMode = .lines
return node
}
func renderer(_ renderer: SCNSceneRenderer, didUpdate node: SCNNode, for anchor: ARAnchor) {
if let faceAnchor = anchor as? ARFaceAnchor, let faceGeometry = node.geometry as? ARSCNFaceGeometry {
faceGeometry.update(from: faceAnchor.geometry)
facePoseAnalyzer(anchor: faceAnchor)
DispatchQueue.main.async {
self.expressionLabel.text = self.faceExpression
self.expression1.text = String(self.count1)
self.expression2.text = String(self.count2)
self.expression3.text = String(self.count3)
self.expression4.text = String(self.count4)
self.expression5.text = String(self.count5)
self.expression6.text = String(self.count6)
self.extractMost()
}
}
}
func extractMost() {
let structedArray = [
emotions(expression: "smile", count: self.count1),
emotions(expression: "dumbfounded", count: self.count2),
emotions(expression: "surprise", count: self.count3),
emotions(expression: "nyah", count: self.count4),
emotions(expression: "awful", count: self.count5),
emotions(expression: "eye smile", count: self.count6)
]
let sortedArray = structedArray.sorted(by: {$0.count > $1.count})
self.resultLabel.text = sortedArray[0].expression
}
func facePoseAnalyzer(anchor: ARFaceAnchor) {
let smileLeft = anchor.blendShapes[.mouthSmileLeft]
let smileRight = anchor.blendShapes[.mouthSmileRight]
let innerUp = anchor.blendShapes[.browInnerUp]
let tongue = anchor.blendShapes[.tongueOut]
let eyeBlinkLeft = anchor.blendShapes[.eyeBlinkLeft]
let jawOpen = anchor.blendShapes[.jawOpen]
let mouthFrownLeft = anchor.blendShapes[.mouthFrownLeft]
let mouthFrownRight = anchor.blendShapes[.mouthFrownRight]
var newFaceExpression = ""
if ((smileLeft?.decimalValue ?? 0.0) + (smileRight?.decimalValue ?? 0.0)) > 0.9 {
newFaceExpression = "😀"
self.count1 = self.count1 + 1
}
if ((jawOpen?.decimalValue ?? 0.0) + (innerUp?.decimalValue ?? 0.0)) > 0.85 {
newFaceExpression = "😧"
self.count2 = self.count2 + 1
}
if innerUp?.decimalValue ?? 0.0 > 0.8 {
newFaceExpression = "😳"
self.count3 = self.count3 + 1
}
if tongue?.decimalValue ?? 0.0 > 0.08 {
newFaceExpression = "😛"
self.count4 = self.count4 + 1
}
if (mouthFrownLeft?.decimalValue ?? 0.0) > 0.3 || (mouthFrownRight?.decimalValue ?? 0.0) > 0.3 {
newFaceExpression = "🤢"
self.count5 = self.count5 + 1
}
if eyeBlinkLeft?.decimalValue ?? 0.0 > 0.5 {
newFaceExpression = "😊"
self.count6 = self.count6 + 1
}
if self.faceExpression != newFaceExpression {
self.faceExpression = newFaceExpression
}
// if cheekPuff?.decimalValue ?? 0.0 > 0.5 {
// newFaceExpression = "🤢"
// self.count5 = self.count5 + 1
// }
}
func session(_ session: ARSession, didFailWithError error: Error) {
// Present an error message to the user
}
func sessionWasInterrupted(_ session: ARSession) {
// Inform the user that the session has been interrupted, for example, by presenting an overlay
}
func sessionInterruptionEnded(_ session: ARSession) {
// Reset tracking and/or remove existing anchors if consistent tracking is required
}
}
<?xml version="1.0" encoding="UTF-8"?>
<document type="com.apple.InterfaceBuilder3.CocoaTouch.XIB" version="3.0" toolsVersion="16097" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" useTraitCollections="YES" useSafeAreas="YES" colorMatched="YES">
<device id="retina6_1" orientation="landscape" appearance="light"/>
<dependencies>
<plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="16087"/>
<capability name="Safe area layout guides" minToolsVersion="9.0"/>
<capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
</dependencies>
<objects>
<placeholder placeholderIdentifier="IBFilesOwner" id="-1" userLabel="File's Owner"/>
<placeholder placeholderIdentifier="IBFirstResponder" id="-2" customClass="UIResponder"/>
<view contentMode="scaleToFill" id="m5W-Np-e6E">
<rect key="frame" x="0.0" y="0.0" width="896" height="414"/>
<autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMaxY="YES"/>
<subviews>
<button opaque="NO" contentMode="scaleToFill" contentHorizontalAlignment="center" contentVerticalAlignment="center" buttonType="roundedRect" lineBreakMode="middleTruncation" translatesAutoresizingMaskIntoConstraints="NO" id="2IZ-Qo-Rl0">
<rect key="frame" x="425" y="192" width="46" height="30"/>
<state key="normal" title="Button"/>
</button>
</subviews>
<color key="backgroundColor" systemColor="systemBackgroundColor" cocoaTouchSystemColor="whiteColor"/>
<constraints>
<constraint firstItem="2IZ-Qo-Rl0" firstAttribute="centerY" secondItem="m5W-Np-e6E" secondAttribute="centerY" id="Xpn-17-T2q"/>
<constraint firstItem="2IZ-Qo-Rl0" firstAttribute="centerX" secondItem="m5W-Np-e6E" secondAttribute="centerX" id="xUd-ag-9LJ"/>
</constraints>
<viewLayoutGuide key="safeArea" id="1md-eY-o7C"/>
<point key="canvasLocation" x="139" y="154"/>
</view>
</objects>
</document>
//
// ViewController.swift
// extract_facial_expression
//
// Created by Jerry kim on 2020/06/11.
// Copyright © 2020 hoya. All rights reserved.
//
import UIKit
class ViewController: UIViewController {
override func viewDidLoad() {
super.viewDidLoad()
// Do any additional setup after loading the view.
}
}