Phone Farm (iClick) - Enterprise iOS Automation Without Jailbreak

SDK Quick Installation

Node.js, Python, and C# users can directly install via package managers without manually implementing clients:

Node.js: npm install iclick-auto
Python: pip install iclick-auto
C#: Install-Package iclick-auto

All these packages implement API calls, event listening, and MetaData binary stream parsing. Other Languages The following code examples may not be fully verified and are for reference and learning purposes only. Please ensure thorough testing and validation before using in production environments.

JavaScript
Java
Go

const wserverPort = 23188
const wsevents = {}
let wsclient = null
let reconnectTimer = null

export function reconnect() {
    if( reconnectTimer || wsclient?.readyState === WebSocket.CONNECTING ){
        return
    }
    reconnectTimer = setTimeout(() => {
        console.log('API service reconnecting...')
        connectServer().finally(() => reconnectTimer = null)
    }, 1000 * 3)
}

export function connectServer() {
    return new Promise((_resolve) => {
        
        if( wsclient ){
            return
        }

        wsclient = new WebSocket(`ws://127.0.0.1:${wserverPort}`)
        wsclient.binaryType = 'arraybuffer'

        wsclient.onopen = () => {
            console.log('API service connected successfully')
            _resolve()
        }

        wsclient.onmessage = (_message) => {
     
            let _payload = _message.data,
                _bindata = null

            if( typeof _payload === 'string' ){

                _payload = JSON.parse(_payload)

            }else{

                // Some interfaces return binary data, need to parse [metaDataLength][metaData][binary]
                const _decoder = new TextDecoder('utf-8')   
                const _metaLength = _decoder.decode(_payload.slice(0, 6))
                                    _bindata = _payload.slice(6 + parseInt(_metaLength))
                                    _payload = JSON.parse(_decoder.decode(_payload.slice(6, 6 + parseInt(_metaLength))))
                _payload.data = _bindata
                _bindata = null
            }
            
            const _eventId = _payload.evtid
            const _event = wsevents[ _eventId ]
  
            if( _event ){

                if( _payload.type === 'error' ){
                    _event.reject(new Error(_payload.error))
                }else{
                    _event.resolve( _payload.data )
                }

                delete wsevents[ _eventId ]

            }else if( _payload.event ){

                // You can dispatch events according to different languages
            }
        }
        
        wsclient.onclose = (code) => {

            wsclient = null
            console.log('API service connection closed:', code)
            reconnect()
        }
        
        wsclient.onerror = (err) => {
            console.error('API service connection error:', err)
        }
    })
}


export function apiInvoke(type, _params = {}, _timeout = 18) {

    return new Promise((resolve, reject) => {

        if( !wsclient || wsclient.readyState !== WebSocket.OPEN ){

            reject( new Error('API service connection not ready') )

        }else{

            const _eventId = Math.random().toString(36).substring(2, 12)
            const _payload = {
                ..._params,
                type,
                evtid: _eventId,
                timeout: _timeout
            }

            wsevents[_payload.evtid] = { resolve, reject }

            wsclient.send(JSON.stringify(_payload))

            if( _timeout > 0 ){
                setTimeout(() => {

                    reject(new Error(`API service: ${_payload.evtid} Invoke Timeout !`))

                    delete wsevents[_payload.evtid]

                }, _timeout * 1000)
            }
        }
    })
}

// Usage example
await connectServer()
const result = await apiInvoke('click', { x: 100, y: 200, deviceId: 'XXXXXXXXXX' })

import org.java_websocket.client.WebSocketClient;
import org.java_websocket.handshake.ServerHandshake;
import com.google.gson.Gson;
import com.google.gson.JsonObject;
import java.net.URI;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.security.SecureRandom;

public class IClient {
    private static final int PORT = 23188;
    private static final SecureRandom RANDOM = new SecureRandom();
    private static final String CHARS = "abcdefghijklmnopqrstuvwxyz0123456789";
    
    private WebSocketClient wsClient;
    private final ConcurrentHashMap<String, CompletableFuture<Object>> events = new ConcurrentHashMap<>();
    private final Gson gson = new Gson();
    private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor();
    private final AtomicBoolean isReconnecting = new AtomicBoolean(false);
    
    public synchronized void connect() throws Exception {
        // Close old connection
        if (wsClient != null) {
            wsClient.close();
        }
        
        URI uri = new URI("ws://127.0.0.1:" + PORT);
        wsClient = new WebSocketClient(uri) {
            @Override
            public void onOpen(ServerHandshake handshake) {
                System.out.println("API service connected successfully");
                isReconnecting.set(false);
            }
            
            @Override
            public void onMessage(String message) {
                handleMessage(gson.fromJson(message, JsonObject.class));
            }
            
            @Override
            public void onMessage(ByteBuffer bytes) {
                byte[] data = bytes.array();
                int metaLength = Integer.parseInt(new String(data, 0, 6, StandardCharsets.UTF_8).trim());
                String metaJson = new String(data, 6, metaLength, StandardCharsets.UTF_8);
                handleMessage(gson.fromJson(metaJson, JsonObject.class));
            }
            
            @Override
            public void onClose(int code, String reason, boolean remote) {
                System.out.println("API service connection closed: " + code);
                reconnect();
            }
            
            @Override
            public void onError(Exception ex) {
                System.err.println("API service connection error: " + ex.getMessage());
            }
        };
        
        wsClient.connect();
    }
    
    private void handleMessage(JsonObject payload) {
        if (!payload.has("evtid")) return;
        
        String evtid = payload.get("evtid").getAsString();
        CompletableFuture<Object> future = events.remove(evtid);
        
        if (future != null) {
            if (payload.has("type") && "error".equals(payload.get("type").getAsString())) {
                future.completeExceptionally(new Exception(payload.get("error").getAsString()));
            } else {
                future.complete(payload.get("data"));
            }
        }
    }
    
    private void reconnect() {
        if (!isReconnecting.compareAndSet(false, true)) return;
        
        scheduler.schedule(() -> {
            try {
                System.out.println("API service reconnecting...");
                connect();
            } catch (Exception e) {
                System.err.println("Reconnection failed: " + e.getMessage());
                isReconnecting.set(false);
            }
        }, 3, TimeUnit.SECONDS);
    }
    
    private static String generateEventId() {
        StringBuilder sb = new StringBuilder(10);
        for (int i = 0; i < 10; i++) {
            sb.append(CHARS.charAt(RANDOM.nextInt(CHARS.length())));
        }
        return sb.toString();
    }
    
    public CompletableFuture<Object> invoke(String type, Map<String, Object> params, int timeout) {
        CompletableFuture<Object> future = new CompletableFuture<>();
        
        if (wsClient == null || !wsClient.isOpen()) {
            future.completeExceptionally(new Exception("API service connection not ready"));
            return future;
        }
        
        String evtid = generateEventId();
        Map<String, Object> payload = new HashMap<>(params != null ? params : new HashMap<>());
        payload.put("type", type);
        payload.put("evtid", evtid);
        payload.put("timeout", timeout);
        
        events.put(evtid, future);
        wsClient.send(gson.toJson(payload));
        
        // Set timeout
        scheduler.schedule(() -> {
            CompletableFuture<Object> f = events.remove(evtid);
            if (f != null) {
                f.completeExceptionally(new TimeoutException("API service timeout"));
            }
        }, timeout, TimeUnit.SECONDS);
        
        return future;
    }
    
    public void close() {
        scheduler.shutdown();
        if (wsClient != null) {
            wsClient.close();
        }
    }
    
    // Usage example
    public static void main(String[] args) throws Exception {
        IClient client = new IClient();
        client.connect();
        Thread.sleep(1000);
        
        Map<String, Object> params = new HashMap<>();
        params.put("x", 100);
        params.put("y", 200);
        params.put("deviceId", "XXXXXXXXXX");
        
        client.invoke("click", params, 18)
            .thenAccept(result -> System.out.println("Click result: " + result))
            .exceptionally(ex -> {
                System.err.println("Call failed: " + ex.getMessage());
                return null;
            })
            .thenRun(client::close);
    }
}

package main

import (
    "encoding/json"
    "fmt"
    "math/rand"
    "sync"
    "time"
    
    "github.com/gorilla/websocket"
)

type IClient struct {
    port            int
    conn            *websocket.Conn
    events          map[string]chan Response
    eventsMutex     sync.RWMutex
    reconnectMutex  sync.Mutex
    isReconnecting  bool
}

type Response struct {
    Data  interface{}
    Error error
}

type Payload struct {
    Type    string                 `json:"type,omitempty"`
    EvtID   string                 `json:"evtid,omitempty"`
    Timeout int                    `json:"timeout,omitempty"`
    Event   string                 `json:"event,omitempty"`
    Data    interface{}            `json:"data,omitempty"`
    Error   string                 `json:"error,omitempty"`
}

func NewIClient() *IClient {
    return &IClient{
        port:   23188,
        events: make(map[string]chan Response),
    }
}

func (c *IClient) Connect() error {
    // Close old connection
    if c.conn != nil {
        c.conn.Close()
    }
    
    url := fmt.Sprintf("ws://127.0.0.1:%d", c.port)
    
    conn, _, err := websocket.DefaultDialer.Dial(url, nil)
    if err != nil {
        return fmt.Errorf("connection failed: %w", err)
    }
    
    c.conn = conn
    fmt.Println("API service connected successfully")
    
    go c.messageHandler()
    
    return nil
}

func (c *IClient) messageHandler() {
    defer func() {
        fmt.Println("API service connection closed")
        c.reconnect()
    }()
    
    for {
        messageType, message, err := c.conn.ReadMessage()
        if err != nil {
            return
        }
        
        var payload Payload
        
        switch messageType {
        case websocket.TextMessage:
            if err := json.Unmarshal(message, &payload); err != nil {
                fmt.Printf("JSON parse error: %v\n", err)
                continue
            }
        case websocket.BinaryMessage:
            // Parse binary message
            var metaLength int
            fmt.Sscanf(string(message[:6]), "%d", &metaLength)
            if err := json.Unmarshal(message[6:6+metaLength], &payload); err != nil {
                fmt.Printf("Metadata parse error: %v\n", err)
                continue
            }
            payload.Data = message[6+metaLength:]
        }
        
        c.processPayload(payload)
    }
}

func (c *IClient) processPayload(payload Payload) {
    if payload.EvtID != "" {
        c.eventsMutex.RLock()
        ch, exists := c.events[payload.EvtID]
        c.eventsMutex.RUnlock()
        
        if exists {
            response := Response{Data: payload.Data}
            if payload.Type == "error" {
                response.Error = fmt.Errorf("%s", payload.Error)
            }
            
            ch <- response
            
            c.eventsMutex.Lock()
            delete(c.events, payload.EvtID)
            c.eventsMutex.Unlock()
        }
    }
}

func (c *IClient) reconnect() {
    c.reconnectMutex.Lock()
    if c.isReconnecting {
        c.reconnectMutex.Unlock()
        return
    }
    c.isReconnecting = true
    c.reconnectMutex.Unlock()
    
    time.Sleep(3 * time.Second)
    fmt.Println("API service reconnecting...")
    
    if err := c.Connect(); err != nil {
        fmt.Printf("Reconnection failed: %v\n", err)
    }
    
    c.reconnectMutex.Lock()
    c.isReconnecting = false
    c.reconnectMutex.Unlock()
}

func (c *IClient) generateEventID() string {
    return fmt.Sprintf("%d%03d", time.Now().UnixNano()%1000000000, rand.Intn(1000))
}

func (c *IClient) Invoke(apiType string, params map[string]interface{}, timeout int) (interface{}, error) {
    if c.conn == nil {
        return nil, fmt.Errorf("API service connection not ready")
    }
    
    evtID := c.generateEventID()
    payload := map[string]interface{}{
        "type":    apiType,
        "evtid":   evtID,
        "timeout": timeout,
    }
    for k, v := range params {
        payload[k] = v
    }
    
    ch := make(chan Response, 1)
    c.eventsMutex.Lock()
    c.events[evtID] = ch
    c.eventsMutex.Unlock()
    
    data, _ := json.Marshal(payload)
    if err := c.conn.WriteMessage(websocket.TextMessage, data); err != nil {
        c.eventsMutex.Lock()
        delete(c.events, evtID)
        c.eventsMutex.Unlock()
        return nil, fmt.Errorf("failed to send message: %w", err)
    }
    
    select {
    case response := <-ch:
        return response.Data, response.Error
    case <-time.After(time.Duration(timeout) * time.Second):
        c.eventsMutex.Lock()
        delete(c.events, evtID)
        c.eventsMutex.Unlock()
        return nil, fmt.Errorf("API service timeout")
    }
}

func (c *IClient) Close() error {
    if c.conn != nil {
        return c.conn.Close()
    }
    return nil
}

// Usage example
func main() {
    client := NewIClient()
    if err := client.Connect(); err != nil {
        fmt.Printf("Connection failed: %v\n", err)
        return
    }
    defer client.Close()
    
    time.Sleep(time.Second)
    
    params := map[string]interface{}{
        "x": 100,
        "y": 200,
        "deviceId": "XXXXXXXXXX",
    }
    
    result, err := client.Invoke("click", params, 18)
    if err != nil {
        fmt.Printf("Call failed: %v\n", err)
        return
    }
    
    fmt.Printf("Click result: %v\n", result)
}

NodeJS Sample Script

How to find clicked UI controls

No-jailbreak scripts can locate controls using image search, color search, and OCR. The iClick console has built-in various color picker, image picker, and OCR development tools.

The script example below demonstrates how to locate the browser on the desktop and access phones.farm.

const iAuto = require('iclick-auto')
const { setTimeout: sleep } = require('node:timers/promises')

;(async () => {
  
    //初始化iClick实例，默认无参数，连接127.0.0.1
    const iclick = new iAuto()
    
    await iclick.connect()
  
    const _deviceId = '1234567890'
    const _device = await iclick.invoke('getDevice', { deviceId: _deviceId })

    if( !_device || !_device.online || !_device.plugged ){
        throw new Error('设备不在线或未就绪')
    }

    console.log('设备信息', _device)

    //返回桌面
    await iclick.invoke('killRecents', { deviceId: _deviceId })

    await sleep(1260)

    //找docker栏的safari
    const _safariIcon = await iclick.invoke('findColor', {
        deviceId: _deviceId,
        colors: [
            [0,0,"FEFEFE|0.95"],
            [25,10,"1FB6FB|0.95"],
            [27,22,"FC3F3D|0.95"],
            [22,26,"FFF8F9|0.95"],
            [35,35,"1E88F8|0.95"]    
        ],
        rect: [0, _screenHeight * 0.7, _screenWidth, _screenHeight] //docker一般在底部
    })
    if( !_safariIcon ){
        throw new Error('未找到docker栏的safari')
    }

    //点击safari
    await iclick.invoke('click', { deviceId: _deviceId, x: _safariIcon.x, y: _safariIcon.y })
    
    //等待进入safari
    await sleep(1260)

    //找到底部刷新按钮
    const _refreshButton = await iclick.invoke('findImage', {
        deviceId: _deviceId,
        sprite: `Qk3aBgAAAAAAADYAAAAoAAAAFgAAA......v7+/v7+AAA=`,
        rect: [0, _screenHeight * 0.7, _screenWidth, _screenHeight] //刷新按钮一般在底部
    })
    if( !_refreshButton ){
        throw new Error('未找到底部刷新按钮')
    }

    //点击刷新按钮左边就是输入框
    await iclick.invoke('click', { deviceId: _deviceId, x: _refreshButton.x - 50, y: _refreshButton.y })

    //等待聚焦
    await sleep(560)

    //输入文字
    await iclick.invoke('sendText', { deviceId: _deviceId, text: 'https://phones.farm' })

    //直接输入回车
    await iclick.invoke('sendKey', { deviceId: _deviceId, key: '', fnkey: 'ENTER' })

    console.log('打开网站成功')

})();

How to simulate getActivity (Recommended)

In traditional Android RPA programming, getting the current activity is a crucial function. It’s used to determine which app or page the device is currently in. However, in jailbreak-free systems, there are no UI node manipulation methods, so naturally, there’s no getActivity function. In iOS instant-cast automation programming, simulating getActivity is essential. It provides the script with nearly 99% fault tolerance and can easily resolve annoying pop-up interference from some apps. Let’s take a simplified version of the video posting script from a certain short video platform as an example.

Find the TK icon on your desktop and click to enter.
Use saveMediaFile to transfer videos from your computer to your phone
Wait for the page to load and click the publish button.
Waiting for the album button to be clicked
Select the file and proceed to the next step.
Use sendText to input and submit content
Wait for the video to finish publishing and then exit the script.

The above process might only require a few dozen lines of code to implement, but the problem isn’t that simple. First, starting from opening the app, aside from the initial authorization pop-up for the photo album and microphone (which only appears once), there are about 10 other business-related pop-ups (with a 30% probability of appearing, and each appearing on a different page). Each of these pop-ups is modal, which will hinder your next action. Simulating getActivity would perfectly solve this problem.

How to simulate getActivity to resolve pop-up windows?

The significance of simulating getActivity is that the page is determined first, and then the operation is performed. Let’s take the TK example above as an example and put it into practice.

//This example uses simplified code for demonstration purposes; please do not copy and run it directly!!!此例子为演示精简了许多代码，请勿直接拷贝运行!!!
const { getDevice } = require('../utils/device')
const asset = require('./asset')
const iclick = require('../utils/iclick')
const { compareColor } = require('../libs')
const { setTimeout: sleep } = require('node:timers/promises')

async function getActivity ({
    device, 
    scenes //scenes 意义在于不同任务指定不同 activity，优化性能
}){

    if( !device ){
        throw new Error('get activity: device not found')
    }
    if( typeof device !== 'object' ){
        device = await getDevice(device)
    }

    const _deviceId = device.info.deviceId
    const _screenWidth = device.info.screenWidth
    const _screenHeight = device.info.screenHeight
    const _tiktokTabbarTop = await asset.get('boundings.tiktok_tabbar_top', device)
    const _dockerTiktok = await iclick.invoke('findColor', {
        deviceId: _deviceId,
        colors: await asset.get('colors.docker_tiktok_icon', device),
    })
    if( _dockerTiktok ){
        return 'desktop'
    }

    if( 
        !scenes.length || 
        scenes.includes('tiktok_home') || 
        scenes.includes('tiktok_friends') || 
        scenes.includes('tiktok_camera_live') || 
        scenes.includes('tiktok_profile') 
    ){

        const _centerTexts = await iclick.invoke('ocr', { deviceId: _deviceId, rect: [ 46, _screenHeight / 4, _screenWidth - 46, _screenHeight - (_screenHeight / 4) ] })
        const _centerContent = _centerTexts.map($ => $.text).join('\n').toLocaleLowerCase()

        //居中弹窗 1
        if( _centerContent.includes('save') && _centerContent.includes('login') && _centerContent.includes('next') && _centerContent.includes('time') ){

            const _notNow = _centerTexts.find($ => $.text.toLowerCase().includes('not') && $.text.toLowerCase().includes('now'))
            if( _notNow ){

                await iclick.invoke('click', { deviceId: _deviceId, x: _notNow.x, y: _notNow.y })
                      
                await sleep(1280)
                return get({ device, scenes })
            }
        }

        //居中弹窗 2
        if( _centerContent.includes('this') && _centerContent.includes('account') && _centerContent.includes('is') && _centerContent.includes('private') && _centerContent.includes('approval') && _centerContent.includes('got') ){

            const _gotIt = _centerTexts.find($ => $.text.toLowerCase().includes('got') && $.text.toLowerCase().includes('it'))
            if( _gotIt ){
                await iclick.invoke('click', { deviceId: _deviceId, x: _gotIt.x, y: _gotIt.y, focus: true })
                      iclick.invoke('mouseReset', { deviceId: _deviceId })
                await sleep(1280)
                return get({ device, scenes })
            }
        }

        //后面大约还有居中弹窗 3-6 个，这里不展开了

        const _bottomTexts = await iclick.invoke('ocr', { deviceId: _deviceId, rect: [ 8, _screenHeight / 2, _screenWidth - 8, _screenHeight - 18 ] })
        const _bottomContent = _bottomTexts.map($ => $.text).join('\n').toLocaleLowerCase()

        //底部弹窗 1
        if( _bottomContent.includes('you') && _bottomContent.includes('history') && _bottomContent.includes('turned') && _bottomContent.includes('viewed') ){

            await iclick.invoke('click', { deviceId: _deviceId, x: _screenWidth / 2, y: 158 })

            await sleep(1280)
            return get({ device, scenes })
        }

        //后面大约还有底部弹窗 1-3 个，这里不展开了
    }



    //先获取底部大概颜色
    const _bottomColor = await iclick.invoke('getColor', { deviceId: _deviceId, x: _screenWidth - 8, y: _tiktokTabbarTop + 15 })

    //黑底
    if( compareColor(_bottomColor, '000000', 0.95) ){

        if( !scenes.length || scenes.includes('tiktok_home') ){

            const _homeActived = await iclick.invoke('findImages', {
                deviceId: _deviceId,
                sprites: await asset.get('sprites.tiktok_home_actived', device),
                rect: [0, _tiktokTabbarTop, _screenWidth / 2, _screenHeight],
            })
            if( _homeActived ){
                return 'tiktok_home'
            }
        }

        if( !scenes.length || scenes.includes('tiktok_friends') ){

            const _friendsActived = await iclick.invoke('findImages', {
                deviceId: _deviceId,
                sprites: await asset.get('sprites.tiktok_friends_actived', device),
                rect: [0, _tiktokTabbarTop, _screenWidth / 2, _screenHeight],
            })
            if( _friendsActived ){
                return 'tiktok_friends'
            }
        }

        if( !scenes.length || scenes.includes('tiktok_camera_live') ){

            const _cameraLive = await iclick.invoke('findColor', {
                deviceId: _deviceId,
                colors: await asset.get('features.tiktok_camera_live', device),
            })
            if( _cameraLive ){
                return 'tiktok_camera_live'
            }else{

                const _cameraLive = await iclick.invoke('findColor', {
                    deviceId: _deviceId,
                    colors: await asset.get('features.tiktok_camera_live2', device),
                })
                if( _cameraLive ){
                    return 'tiktok_camera_live'
                }
            }
        }

        if( !scenes.length || scenes.includes('tiktok_creative_confirm') ){

            const _creativeConfirmNext = await iclick.invoke('findImage', {
                deviceId: _deviceId,
                sprite: await asset.get('sprites.tiktok_creative_confirm_next', device),
                rect: [_screenWidth / 2, _screenHeight - 280, _screenWidth, _screenHeight - 18],
            })
            if( _creativeConfirmNext ){
                return 'tiktok_creative_confirm'
            }
        }
    }
    //白底
    else if( compareColor(_bottomColor, 'ffffff', 0.95) ){

        if( !scenes.length || scenes.includes('tiktok_profile') ){

            const _profileActived = await iclick.invoke('findImages', {
                deviceId: _deviceId,
                sprites: await asset.get('sprites.tiktok_profile_actived', device),
                rect: [_screenWidth / 2, _tiktokTabbarTop, _screenWidth, _screenHeight],
            })
            if( _profileActived ){
                return 'tiktok_profile'
            }
        }

        if( !scenes.length || scenes.includes('tiktok_friends') ){

            const _friendsActived = await iclick.invoke('findImages', {
                deviceId: _deviceId,
                sprites: await asset.get('sprites.tiktok_friends_actived', device),
                rect: [0, _tiktokTabbarTop, _screenWidth / 2, _screenHeight],
            })
            if( _friendsActived ){
                return 'tiktok_friends'
            }
        }

        if( !scenes.length || scenes.includes('tiktok_filepicker') ){

            const _filepickerRecents = await iclick.invoke('findImage', {
                deviceId: _deviceId,
                sprite: await asset.get('sprites.tiktok_filepicker_recents', device),
                rect: [158, 18, _screenWidth - 158, 280],
            })
            if( _filepickerRecents ){
                return 'tiktok_filepicker'
            }
        }


        if( !scenes.length || scenes.includes('tiktok_creative_editor') ){

            const _creativeSavePost = await iclick.invoke('findImage', {
                deviceId: _deviceId,
                sprite: await asset.get('sprites.tiktok_creative_save_post', device),
                rect: [_screenWidth / 2, _screenHeight - 280, _screenWidth, _screenHeight - 18],
            })
            if( _creativeSavePost ){
                return 'tiktok_creative_editor'
            }
        }

        //Tiktok activity 不止这些，后面不展开了
    }

    return 'unknown'
}

The code above simulated getActivity and solved the pop-up issue. Now we’ll write the “posting script” code.

//This example uses simplified code for demonstration purposes; please do not copy and run it directly!!!此例子为演示精简了许多代码，请勿直接拷贝运行!!!
const { setTimeout: sleep } = require('node:timers/promises')
const { getDevice } = require('../../utils/device')
const asset = require('../asset')
const { goHome } = require('./')
const iclick = require('../../utils/iclick')
const activity = require('../activity')
const { getOneMaterial, deleteMaterial } = require('../../utils/material')

async function autoPublish({ deviceId }) {

    const _device = await getDevice(deviceId)
    const _deviceId = _device.info.deviceId
    const _screenWidth = _device.info.screenWidth
    const _screenHeight = _device.info.screenHeight

    if( !_device.online || !_device.plugged ){
        throw new Error(`设备 ${_deviceId} 不在线或未连接`)
    }

    await goHome(_device)

    const _runtime = {
        publisheds: 0,
        activity_lose_count: 0,
        material: await getOneMaterial(_device),
        filepicker_lb_clicks: 0,
        material: { content: 'test', fileName: 'D:\\test.mp4' }
    }
    const _publishTimeout = 300

    //发作品只会用到的页面
    const _activsces = [
        'tiktok_home', 
        'tiktok_camera_live',
        'tiktok_filepicker',
        'tiktok_creative_editor',
        'tiktok_friends',
        'tiktok_passport',
        'tiktok_inbox'
    ]

    try{

        //清理最近任务
        await iclick.invoke('killRecents', { deviceId: _deviceId })

        while( true ){

            //获取当前页面
            const _activity = await activity.get({
                device: _device,
                scenes: _activsces
            })

            if( _activity !== 'unknown' ){
                _runtime.activity_lose_count = 0
            }else{
                if( !_runtime.activity_lose_count ){
                    _runtime.activity_lose_count = 0
                }
                _runtime.activity_lose_count ++
            }
    
            if( _runtime.activity_lose_count >= 18 ){
                throw new Error('未适配的页面')
            }

            if( _activity === 'desktop' ){

                const _tkicon = await iclick.invoke('findColor', {
                    deviceId: _deviceId,
                    colors: await asset.get('colors.docker_tiktok_icon', _device),
                })
            
                if( _tkicon ){

                    await iclick.invoke('clearMediaFile', { deviceId: _deviceId })
                    await iclick.invoke('saveMediaFile', { deviceId: _deviceId, file: _runtime.material.fileName }, 0)

                    await iclick.invoke('click', { deviceId: _deviceId, x: _tkicon.x, y: _tkicon.y })

                    //等待进入TK，这里久一些
                    await iclick.invoke('mouseReset', { deviceId: _deviceId })
                    await sleep(1560)
                    
                }else{
    
                    throw new Error('未在桌面Docker找到TK图标')
                }
            }
            else if( _activity === 'tiktok_home' || _runtime.forceInTiktokHome )
            {
    
                if( !_runtime.pushlishTime ){

                    await iclick.invoke('click', { deviceId: _deviceId, ...(await asset.get('boundings.tiktok_creative_button', _device)) })

                }else{

                    const _texts = await iclick.invoke('ocr', {deviceId: _deviceId, rect: [18, 18, _screenWidth - 18, 188]})
                    const _content = _texts.map($ => $.text.toLocaleLowerCase()).join('\n')
             
                    if( _content.includes('video posted') || (+new Date / 1000 - _runtime.pushlishTime) >= _publishTimeout ){
        
                        console.log('video posted')
        
                        await deleteMaterial(_device, _runtime.material.id)
    
                        _runtime.pushlishTime = 0
                        _runtime.material = await getOneMaterial(_device)
                        _runtime.activity_lose_count = 0
                        _runtime.publisheds ++
                        _runtime.forceInTiktokHome = false
    
                        console.log('video posted')

                        return true
                    }
                }
            }
            else if( _activity === 'tiktok_camera_live' )
            {

                if( _runtime.filepicker_lb_clicks >= 3 ){
                    await iclick.invoke('click', { deviceId: _deviceId, ...(await asset.get('boundings.tiktok_camera_live_picker2', _device)) }) 
                }else{
                    await iclick.invoke('click', { deviceId: _deviceId, ...(await asset.get('boundings.tiktok_camera_live_picker', _device)) })
                    _runtime.filepicker_lb_clicks++
                }
            }
            else if( _activity === 'tiktok_filepicker' )
            {
                
                _runtime.filepicker_lb_clicks = 0

                //先找到文件picker的navbar，然后逐步点击
                const _navbarBottom = await iclick.invoke('findImage', {
                    deviceId: _deviceId,
                    sprite: await asset.get('sprites.tiktok_filepicker_navbar_actived', _device),
                    rect: [0, 18, _screenWidth / 2, 218],
                })

                if( _navbarBottom ){

                    await iclick.invoke('click', { deviceId: _deviceId, x: 18, y: await asset.get('boundings.tiktok_creative_filepicker_first_top', _device) })

                    let _retrys = 1
                    while( true ){

                        if( _retrys > 5 ){
                            throw new Error('进入编辑器失败')
                        }

                        const _activnow = await activity.get({ device: _device, scenes: _activsces })
                        if( _activnow === 'tiktok_creative_editor' ){
                            break
                        }

                        await iclick.invoke('click', { deviceId: _deviceId, ...(await asset.get('boundings.tiktok_creative_confirm_next', _device)) })
                        await sleep(2560)
                        _retrys++
                    }

                }else{
                    throw new Error('未找到文件picker的navbar')
                }
            }
            else if( _activity === 'tiktok_creative_editor' )
            {

                if( _runtime.material.content ){
                    if( _runtime.material.type === 'image' ){

                        await iclick.invoke('click', { deviceId: _deviceId, x: _screenWidth / 2, y: await asset.get('boundings.tiktok_creative_content_top_image', _device) })
    
                        await iclick.invoke('sendText', { deviceId: _deviceId, text: _runtime.material.content })
    
                        await sleep(560)
    
                        const _editQuit = await iclick.invoke('findImage', {
                            deviceId: _deviceId,
                            sprite: await asset.get('sprites.tiktok_creative_imgtext_edit_quit', _device),
                            rect: [0, 0, _screenWidth, _screenHeight],
                        })
    
                        if( _editQuit ){
                            await iclick.invoke('click', { deviceId: _deviceId, x: _editQuit.x, y: _editQuit.y })
                        }else{
                            throw new Error('未找到编辑退出按钮')
                        }
    
                    }else{
    
                        await iclick.invoke('click', { deviceId: _deviceId, x: _screenWidth / 2.3, y: await asset.get('boundings.tiktok_creative_content_top_video', _device) })
    
                        await sleep(260)

                        await iclick.invoke('sendText', { deviceId: _deviceId, text: _runtime.material.content })

                        await sleep(560)
    
                        await iclick.invoke('click', { deviceId: _deviceId, x: _screenWidth / 2, y: _screenHeight / 2 })
                    }
                }

                await sleep(560)

                await iclick.invoke('click', { deviceId: _deviceId, ...(await asset.get('boundings.tiktok_creative_confirm_next', _device)) })

                await sleep(1260)

                const _texts = await iclick.invoke('ocr', {deviceId: _deviceId, rect: [48, _screenHeight * 0.3, _screenWidth - 48, _screenHeight * 0.7]})
                const _content = _texts.map($ => $.text.toLocaleLowerCase()).join('\n')

                if( _content.includes('post') && _content.includes('video') && _content.includes('publi') && _content.includes('setting') ){

                    const _postNow = _texts.find($ => $.text.toLowerCase().trim().includes('post now'))
                    if( _postNow ){

                        await iclick.invoke('click', { deviceId: _deviceId, x: _postNow.x, y: _postNow.y })

                        continue
                    }
                }

                _runtime.pushlishTime = +new Date / 1000
            }
            else if( _activity === 'tiktok_friends' ){
                
                _runtime.forceInTiktokHome = true

            }
            else
            {

            }

            await sleep(1560)
        }

        
    }catch(_error){

        await sleep(1260)

        throw _error
    }
}

The code above is for reference only; many dependency codes have been removed due to space limitations. Referring to the getActivity code above and the actual “post content” script code, it first determines the current page and resolves pop-ups before each operation. Based on this code model, even on older devices like the iPhone 6 where occasional app crashes due to device lag won’t affect the task’s operation and it can continue to run.

Practical role of OCR in automation

OCR is generally used in conjunction with image and color recognition in scenarios where specific logic needs to be distinguished before performing operations. It returns the text and coordinates of a specified area. iClick’s default OCR uses the device’s GPU for recognition, which does not consume host machine performance. However, it is unavailable or unsuitable in some scenarios.

❌iPhone 8 and earlier models	Phones prior to this model do not support neural network acceleration chips.
❌ Scenarios requiring large-scale recognition or document extraction	Due to iOS memory limitations, this is not recommended. Please use PaddleOcr instead.

For scenarios requiring the use of PaddleOcr, the iClick console integrates a one-click PaddleOcr installation function, which can be found in iClick -> Settings -> Development.

Getting Started

API Reference

Multi-language Client

SDK Quick Installation

NodeJS Sample Script

How to find clicked UI controls

How to simulate getActivity (Recommended)

How to simulate getActivity to resolve pop-up windows?

Practical role of OCR in automation

​SDK Quick Installation

​NodeJS Sample Script

​How to find clicked UI controls

​ How to simulate getActivity (Recommended)

​How to simulate getActivity to resolve pop-up windows?

​Practical role of OCR in automation

SDK Quick Installation

NodeJS Sample Script

How to find clicked UI controls

How to simulate getActivity (Recommended)

How to simulate getActivity to resolve pop-up windows?

Practical role of OCR in automation