Using ArcFace ML Kit
2
Hi all, I'm trying to implement ArcFace in my ML-capable Mendix application. I currently run into the following error with my preprocessing Java-Action: Error in execution of monitored action 'RegularClientAction' (execution id: 1698994735141-7, execution type: CLIENT) -------- java.lang.UnsatisfiedLinkError: org.opencv.core.Mat.n_Mat()J at org.opencv.core.Mat.n_Mat(Native Method) at org.opencv.core.Mat.<init>(Mat.java:23) at org.opencv.core.MatOfByte.<init>(MatOfByte.java:34) at arcface.actions.Face_Preprocess.normalizeImage(Face_Preprocess.java:72) at arcface.actions.Face_Preprocess.executeAction(Face_Preprocess.java:47) at arcface.actions.Face_Preprocess.executeAction(Face_Preprocess.java:26) at com.mendix.systemwideinterfaces.core.UserAction.execute(UserAction.java:58) at com.mendix.basis.actionmanagement.CoreActionHandlerImpl.doCall(CoreActionHandlerImpl.scala:71) at com.mendix.basis.actionmanagement.CoreActionHandlerImpl.call(CoreActionHandlerImpl.scala:48) at com.mendix.core.actionmanagement.internal.InternalCoreAction.call(InternalCoreAction.java:57) at com.mendix.basis.actionmanagement.ActionManager.$anonfun$executeSync$2(ActionManager.scala:106) at com.mendix.util.classloading.Runner$.withContextClassLoader(Runner.scala:20) at com.mendix.basis.actionmanagement.ActionManager.executeSync(ActionManager.scala:105) at com.mendix.basis.actionmanagement.UserActionCallBuilderImpl.execute(UserActionCallBuilderImpl.scala:57) at com.mendix.modules.microflowengine.actions.actioncall.ForegroundJavaAction.doExecute(ForegroundJavaAction.scala:35) at com.mendix.modules.microflowengine.actions.actioncall.ForegroundJavaAction.doExecute(ForegroundJavaAction.scala:11) at com.mendix.modules.microflowengine.actions.actioncall.JavaAction.execute(JavaAction.scala:38) at com.mendix.modules.microflowengine.microflow.impl.MicroflowObject.$anonfun$execute$1(MicroflowObject.scala:32) at scala.Option.flatMap(Option.scala:283) at com.mendix.modules.microflowengine.microflow.impl.MicroflowObject.execute(MicroflowObject.scala:29) at com.mendix.modules.microflowengine.microflow.impl.MicroflowImpl.$anonfun$executeAfterBreakingIfNecessary$2(MicroflowImpl.scala:167) at scala.Option.flatMap(Option.scala:283) at com.mendix.modules.microflowengine.microflow.impl.MicroflowImpl.executeAfterBreakingIfNecessary(MicroflowImpl.scala:167) at com.mendix.modules.microflowengine.microflow.impl.MicroflowImpl.executeAction(MicroflowImpl.scala:114) at com.mendix.systemwideinterfaces.core.UserAction.execute(UserAction.java:58) at com.mendix.basis.actionmanagement.CoreActionHandlerImpl.doCall(CoreActionHandlerImpl.scala:71) at com.mendix.basis.actionmanagement.CoreActionHandlerImpl.call(CoreActionHandlerImpl.scala:48) at com.mendix.core.actionmanagement.internal.InternalCoreAction.call(InternalCoreAction.java:57) at com.mendix.basis.actionmanagement.ActionManager.$anonfun$executeSync$2(ActionManager.scala:106) at com.mendix.util.classloading.Runner$.withContextClassLoader(Runner.scala:20) at com.mendix.basis.actionmanagement.ActionManager.executeSync(ActionManager.scala:105) at com.mendix.basis.actionmanagement.MicroflowCallBuilderImpl.execute(MicroflowCallBuilderImpl.scala:62) at com.mendix.webui.actions.client.MicroflowRuntimeOperationExecutor.runMicroflow(MicroflowRuntimeOperationExecutor.scala:97) at com.mendix.webui.actions.client.MicroflowRuntimeOperationExecutor.$anonfun$apply$5(MicroflowRuntimeOperationExecutor.scala:57) at com.mendix.webui.actions.client.RegularClientAction$Helpers$.$anonfun$liftEither$1(RegularClientAction.scala:30) at com.mendix.webui.actions.client.RegularClientAction$Helpers$StateHandler.$anonfun$apply$4(RegularClientAction.scala:56) at com.mendix.webui.requesthandling.helpers.StateHandling.withState(StateHandling.scala:46) at com.mendix.webui.requesthandling.helpers.StateHandling.withState$(StateHandling.scala:43) at com.mendix.webui.actions.client.RegularClientAction$Helpers$StateHandler.withState(RegularClientAction.scala:46) at com.mendix.webui.actions.client.RegularClientAction$Helpers$StateHandler.apply(RegularClientAction.scala:54) at com.mendix.webui.actions.client.RegularClientAction$Helpers$StateHandler.apply(RegularClientAction.scala:46) at com.mendix.webui.actions.client.MicroflowRuntimeOperationExecutor.apply(MicroflowRuntimeOperationExecutor.scala:58) at com.mendix.webui.actions.client.RuntimeOperationAction.$anonfun$apply$1(RuntimeOperationAction.scala:57) at scala.util.Either.flatMap(Either.scala:352) at com.mendix.webui.actions.client.RuntimeOperationAction.apply(RuntimeOperationAction.scala:41) at com.mendix.webui.actions.client.RuntimeOperationAction.apply(RuntimeOperationAction.scala:30) at com.mendix.webui.actions.client.RegularClientAction$Helpers$.$anonfun$liftEither$1(RegularClientAction.scala:30) at com.mendix.webui.actions.client.RegularClientAction.$anonfun$execute$3(RegularClientAction.scala:120) at scala.util.Try$.apply(Try.scala:210) at com.mendix.webui.actions.client.RegularClientAction.$anonfun$execute$2(RegularClientAction.scala:120) at com.mendix.webui.actions.client.RegularClientAction.$anonfun$execute$2$adapted(RegularClientAction.scala:118) at com.mendix.webui.requesthandling.helpers.ContextHandling.$anonfun$inContext$7(ContextHandling.scala:58) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at com.mendix.basis.actionmanagement.ActionMonitoring$.$anonfun$monitor$1(ActionMonitoring.scala:52) at com.mendix.util.classloading.Runner$.withContextClassLoader(Runner.scala:20) at com.mendix.basis.actionmanagement.ActionMonitoring$.monitor(ActionMonitoring.scala:52) at com.mendix.webui.requesthandling.helpers.ContextHandling.inContext(ContextHandling.scala:58) at com.mendix.webui.requesthandling.helpers.ContextHandling.inContext$(ContextHandling.scala:32) at com.mendix.webui.actions.client.RegularClientAction.inContext(RegularClientAction.scala:94) at com.mendix.webui.requesthandling.helpers.ContextHandling.inContext(ContextHandling.scala:29) at com.mendix.webui.requesthandling.helpers.ContextHandling.inContext$(ContextHandling.scala:21) at com.mendix.webui.actions.client.RegularClientAction.inContext(RegularClientAction.scala:94) at com.mendix.webui.actions.client.RegularClientAction.$anonfun$execute$1(RegularClientAction.scala:118) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at com.mendix.webui.requesthandling.helpers.ProfileHandling.profileRequest(ProfileHandling.scala:14) at com.mendix.webui.requesthandling.helpers.ProfileHandling.profileRequest$(ProfileHandling.scala:10) at com.mendix.webui.actions.client.RegularClientAction.profileRequest(RegularClientAction.scala:94) at com.mendix.webui.actions.client.RegularClientAction.execute(RegularClientAction.scala:115) at com.mendix.webui.requesthandling.ClientRequestHandler.handleAction(ClientRequestHandler.scala:105) at com.mendix.webui.requesthandling.ClientRequestHandler.processRequest(ClientRequestHandler.scala:78) at com.mendix.externalinterface.connector.RequestHandler.doProcessRequest(RequestHandler.java:37) at com.mendix.external.connector.MxRuntimeConnector.$anonfun$processRequest$1(MxRuntimeConnector.scala:54) at com.mendix.external.connector.MxRuntimeConnector.$anonfun$processRequest$1$adapted(MxRuntimeConnector.scala:54) at com.mendix.util.classloading.Runner$.withContextClassLoader(Runner.scala:20) at com.mendix.external.connector.MxRuntimeConnector.processRequest(MxRuntimeConnector.scala:54) at com.mendix.basis.impl.MxRuntimeImpl.processRequest(MxRuntimeImpl.scala:221) at com.mendix.m2ee.appcontainer.server.handler.RuntimeServlet.service(RuntimeServlet.scala:40) at javax.servlet.http.HttpServlet.service(HttpServlet.java:750) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:764) at org.eclipse.jetty.servlet.ServletHandler$ChainEnd.doFilter(ServletHandler.java:1665) at org.eclipse.jetty.websocket.servlet.WebSocketUpgradeFilter.doFilter(WebSocketUpgradeFilter.java:170) at org.eclipse.jetty.servlet.FilterHolder.doFilter(FilterHolder.java:202) at org.eclipse.jetty.servlet.ServletHandler$Chain.doFilter(ServletHandler.java:1635) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:527) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1570) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:221) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1384) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:176) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:484) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1543) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:174) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1306) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:129) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:122) at org.eclipse.jetty.server.Server.handle(Server.java:563) at org.eclipse.jetty.server.HttpChannel$RequestDispatchable.dispatch(HttpChannel.java:1598) at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:753) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:501) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:282) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:314) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:100) at org.eclipse.jetty.io.SelectableChannelEndPoint$1.run(SelectableChannelEndPoint.java:53) at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.runTask(AdaptiveExecutionStrategy.java:421) at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.consumeTask(AdaptiveExecutionStrategy.java:390) at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.tryProduce(AdaptiveExecutionStrategy.java:277) at org.eclipse.jetty.util.thread.strategy.AdaptiveExecutionStrategy.run(AdaptiveExecutionStrategy.java:199) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:411) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:969) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.doRunJob(QueuedThreadPool.java:1194) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1149) at java.base/java.lang.Thread.run(Thread.java:834) My Java action is based of the ResNet model, that is found in the ML demo application (GitHub - mendix/mlkit-example-app: Demo for Mendix MLKit). I pass an image to my Java action. Below is the code that I use: // This file was generated by Mendix Studio Pro. // // WARNING: Only the following code will be retained when actions are regenerated: // - the import list // - the code between BEGIN USER CODE and END USER CODE // - the code between BEGIN EXTRA CODE and END EXTRA CODE // Other code you write will be lost the next time you deploy the project. // Special characters, e.g., é, ö, à , etc. are supported in comments. package arcface.actions; import org.opencv.core.Mat; import org.opencv.core.Size; import org.opencv.core.CvType; import org.opencv.imgcodecs.Imgcodecs; import org.opencv.imgproc.Imgproc; import org.opencv.core.MatOfByte; import java.io.ByteArrayOutputStream; import java.io.InputStream; import com.mendix.core.Core; import com.mendix.systemwideinterfaces.core.IContext; import com.mendix.webui.CustomJavaAction; import com.mendix.systemwideinterfaces.core.IMendixObject; import com.mendix.mlkit.MLKit; public class Face_Preprocess extends CustomJavaAction<IMendixObject> { private IMendixObject __FaceImage; private arcface.proxies.TestFace FaceImage; public Face_Preprocess(IContext context, IMendixObject FaceImage) { super(context); this.__FaceImage = FaceImage; } @java.lang.Override public IMendixObject executeAction() throws Exception { this.FaceImage = this.__FaceImage == null ? null : arcface.proxies.TestFace.initialize(getContext(), __FaceImage); // BEGIN USER CODE final ByteArrayOutputStream bos = new ByteArrayOutputStream(); this.FaceImage.getContents(getContext(), bos); byte[] binaryImage = bos.toByteArray(); float[][][][] inputArray = normalizeImage(binaryImage, 112, 112); final InputStream is = MLKit.toInputStream(inputArray); final String b64 = MLKit.toBase64(is); final IMendixObject obj = Core.instantiate(getContext(), "ArcFace.Input_ML_ArcFace"); obj.setValue(getContext(), "Data", b64); return obj; // END USER CODE } /** * Returns a string representation of this action * @return a string representation of this action */ @java.lang.Override public java.lang.String toString() { return "Face_Preprocess"; } // BEGIN EXTRA CODE private float[][][][] normalizeImage(byte[] imageBytes, int height, int width) { // Convert byte array to Mat Mat image = Imgcodecs.imdecode(new MatOfByte(imageBytes), Imgcodecs.IMREAD_COLOR); // Resize the image to the target height and width Size size = new Size(width, height); Imgproc.resize(image, image, size); // Convert image pixel values to float and normalize to range [0, 1] image.convertTo(image, CvType.CV_32F, 1.0 / 255.0); // Rearrange the dimensions to match [1][3][224][224] float[][][][] normalizedImage = new float[1][3][height][width]; for (int i = 0; i < height; i++) { for (int j = 0; j < width; j++) { double[] pixel = image.get(i, j); normalizedImage[0][0][i][j] = (float) pixel[0]; // Channel 1 (R) normalizedImage[0][1][i][j] = (float) pixel[1]; // Channel 2 (G) normalizedImage[0][2][i][j] = (float) pixel[2]; // Channel 3 (B) } } return normalizedImage; } // END EXTRA CODE } Any help or suggestions will be greatly appreciated!
asked
Nico Mouton
1 answers
0
answered
Chris de Gelder