restructure project

.docker/puppeteer/.gitignore

@@ -1,2 +0,0 @@
-node_modules/
-package-lock.json

.docker/puppeteer/Dockerfile

@@ -1,18 +0,0 @@
-FROM linuxserver/rdesktop:alpine-icewm
-# FROM linuxserver/rdesktop:alpine-openbox
-# FROM alpine:3.18
-
-RUN apk update --no-cache
-RUN apk add --no-cache npm chromium
-
-RUN mkdir /puppeteer
-WORKDIR /puppeteer
-
-COPY ./puppeteer.js /puppeteer/puppeteer.js
-COPY ./server.js /puppeteer/server.js
-
-USER root
-COPY ./puppeteer.sh /puppeteer.sh
-RUN chmod +x /puppeteer.sh
-
-CMD [ "sh", "/puppeteer.sh" ]

.docker/puppeteer/puppeteer.sh

@@ -1,4 +0,0 @@
-#!/bin/sh
-sudo chown abc:abc -R /puppeteer/chrome-profiles
-npm i puppeteer
-node server.js

.docker/puppeteer/server.js

@@ -1,55 +0,0 @@
-const http = require('http');
-const url = require('url');
-const fs = require('fs');
-const puppeteer = require('./puppeteer');
-
-// Create an HTTP server
-const server = http.createServer((req, res) => {
-  const parseUrl = url.parse(req.url, true);
-
-  const fail = (error) => {
-    res.writeHead(200, {'Content-Type': 'application/json'});
-    res.end(JSON.stringify({
-      status: false,
-      message: error
-    }))
-  }
-
-  const config = {
-    screenshotPath: './screenshot2.jpg'
-  }
-
-  if (parseUrl.query.url) {
-    config.url = parseUrl.query.url
-  }
-
-
-  try {
-    puppeteer(config)
-      .then(function() {
-        fs.readFile(config.screenshotPath, (err, data) => {
-          if (err) {
-            res.writeHead(500);
-            res.end('Internal Server Error');
-          } else {
-            res.writeHead(200, {'Content-Type': 'image/jpeg'});
-            res.end(data);
-          }
-        });
-      })
-      .catch(function(error) {
-        fail(error.toString())
-      });
-  } catch (error) {
-    fail(error.toString())
-  }
-});
-
-// Set the port to listen to
-const port = process.env.PORT || 4000;
-const host = '0.0.0.0'; // Listen on all network interfaces
-
-// Start the server
-server.listen(port, host, () => {
-  console.log(`Server running at http://${host}:${port}/`);
-});

.docker/rabbitmq/.gitignore

@@ -0,0 +1,2 @@
+data/
+log/

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "svg.preview.background": "editor"
+}

README.md

@@ -3,7 +3,7 @@
 
 You should change the /src/.env.example file name to .env and run
 ```
-docker network create laravel
+docker network create producer
 docker compose up rabbitmq socketio -d
 ```
 
@@ -32,7 +32,7 @@ Version: 1
 http://localhost:8001/horizon
 ```
 
-Rabbitmq: rabbit/rabbit@123
+Rabbitmq: rabbit/rabbit123
 ```
 http://localhost:15672
 ```

consumer/.gitignore

@@ -0,0 +1 @@
+node_modules/

consumer/Dockerfile

@@ -0,0 +1,19 @@
+# FROM linuxserver/rdesktop:alpine-icewm
+# # FROM linuxserver/rdesktop:alpine-openbox
+FROM alpine:3.18
+
+RUN apk update --no-cache
+RUN apk add --no-cache npm chromium
+
+RUN mkdir /consumer
+WORKDIR /consumer
+
+COPY ./consumer.js /consumer/consumer.js
+COPY ./puppeteer.js /consumer/puppeteer.js
+COPY ./package.json /consumer/package.json
+
+USER root
+COPY ./docker.sh /docker.sh
+RUN chmod +x /docker.sh
+
+CMD [ "sh", "/docker.sh" ]

consumer/consumer.js

@@ -0,0 +1,47 @@
+const amqplib = require("amqplib");
+const connectStr = "amqp://rabbit:rabbit123@localhost";
+
+(async () => {
+  const conn = await amqplib.connect(connectStr);
+  const channel = await conn.createChannel();
+  const queue = "process1";
+
+  process.once("SIGINT", async () => {
+    await channel.close();
+    await connection.close();
+  });
+
+  await channel.assertQueue(queue);
+  await channel.prefetch(1);
+
+  await channel.consume(
+    queue,
+    async (message) => {
+      try {
+        await processMessage(channel, message);
+      } catch (error) {
+        console.error("Error processing message:", error.message);
+        // If processing fails, you can choose to requeue or acknowledge the message
+        // Here, we're acknowledging the message even if an error occurs to remove it from the queue
+        channel.ack(message);
+      }
+    },
+    {noAck: false}
+  );
+
+  console.log("[*] Waiting for messages. To exit press CTRL+C");
+})();
+
+
+async function processMessage(channel, message) {
+  try {
+    const response = JSON.parse(message.content.toString())
+    console.log("[x] Received", response);
+    await require('./puppeteer')({})
+    console.log("[x] Done");
+    // After successful processing, acknowledge the message to delete it from the queue
+    channel.ack(message);
+  } catch (error) {
+    throw new Error("Error during processing: " + error.message);
+  }
+}

consumer/docker.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+# sudo chown abc:abc -R /puppeteer/chrome-profiles
+npm i
+node consumer.js

consumer/package.json

@@ -0,0 +1,7 @@
+{
+  "dependencies": {
+    "amqplib": "^0.10.3",
+    "bramqp": "^0.6.1",
+    "puppeteer": "^22.3.0"
+  }
+}

consumer/puppeteer.js

@@ -3,15 +3,14 @@ const puppeteer = require('puppeteer');
 const launch = async (
   config = { url: '', screenshotPath: '', executablePath: '', timeout: 1000 }
 ) => {
-
   config = Object.assign({
-    url: 'https://scrapingbee.com',
+    url: 'https://www.timeanddate.com/',
-    screenshotPath: './screenshot.jpg',
+    screenshotPath: `./screenshot/${Date.now()}.jpg`,
     executablePath: '/usr/bin/chromium',
     timeout: 1000,
   }, config)
 
-  const profile = {
+  const PROFILE = {
     data: [
       // 'wormlazy199210',
       'wormlazy199211',
@@ -28,7 +27,7 @@ const launch = async (
       '--no-sandbox',
       '--headless',
       '--disable-gpu',
-      `--user-data-dir=/puppeteer/chrome-profiles/${profile.select()}`
+      `--user-data-dir=/puppeteer/chrome-profiles/${PROFILE.select()}`
     ]
   });
   const page = await browser.newPage();
@@ -51,6 +50,9 @@ const launch = async (
     console.log(`Error: ${err.message}`);
   } finally {
     await browser.close();
+    await Object.assign(config, {
+      status: true,
+    })
   }
 };
 module.exports = launch

docker-compose.yml

@@ -3,42 +3,28 @@ version : '3'
 name: laravel-rabbitmq${VERSION}
 
 networks:
-  laravel:
+  producer:
     external: true
-  puppeteer:
+  rabbitmq:
+    driver: bridge
+  socketio:
     driver: bridge
   redis:
     driver: bridge
 
 services:
-  app:
+  consumer:
-    tty: true
-    image: bitnami/laravel:9
-    volumes:
-      - ./src:/app
-    command: "sh /app/laravel-entrypoint.sh"
-    ports:
-      - 800${VERSION}:8000
-    networks:
-      - laravel
-      - puppeteer
-      - redis
-    depends_on:
-      - redis
-      - puppeteer
-
-  puppeteer:
     build:
-      context: ./.docker/puppeteer
+      context: ./consumer
       dockerfile: Dockerfile
     shm_size: '1GB'
     volumes:
-      - ./chrome-profiles:/puppeteer/chrome-profiles
+      - ./chrome-profiles:/consumer/chrome-profiles
     ports:
       - :3389
       - :4000
     networks:
-      - puppeteer
+      - rabbitmq
     privileged: true
     restart: always
 
@@ -50,6 +36,21 @@ services:
       - redis
 
   # ----COMMON---- #
+  producer:
+    tty: true
+    image: bitnami/laravel:9
+    volumes:
+      - ./producer:/producer
+    command: "sh /producer/producer.sh"
+    ports:
+      - 800${VERSION}:8000
+    networks:
+      - rabbitmq
+      - socketio
+      - redis
+    depends_on:
+      - redis
+
   socketio:
     container_name: socketio
     build:
@@ -58,7 +59,7 @@ services:
     ports:
       - 3000:3000
     networks:
-      - laravel
+      - socketio
 
   rabbitmq:
     image: rabbitmq:3-management
@@ -69,9 +70,9 @@ services:
       - 15672:15672
     environment:
       - RABBITMQ_DEFAULT_USER=rabbit
-      - RABBITMQ_DEFAULT_PASS=rabbit@123
+      - RABBITMQ_DEFAULT_PASS=rabbit123
     volumes:
       - ./.docker/rabbitmq/data:/var/lib/rabbitmq/
       - ./.docker/rabbitmq/log:/var/log/rabbitmq
     networks:
-      - laravel
+      - rabbitmq

producer/.env.example

@@ -63,7 +63,7 @@ VITE_PUSHER_APP_CLUSTER="${PUSHER_APP_CLUSTER}"
 RABBITMQ_HOST=rabbitmq
 RABBITMQ_PORT=5672
 RABBITMQ_USER=rabbit
-RABBITMQ_PASSWORD=rabbit@123
+RABBITMQ_PASSWORD=rabbit123
 RABBITMQ_VHOST=/
 
 HORIZON_QUEUES='["default", "process1", "process2", "process3"]'

producer/producer.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+php -v
+
+# COMPOSER
+if [ ! -d "/producer/vendor" ]
+then
+    composer install
+fi
+
+# DATABASE
+if [ ! -f "/producer/database/database.sqlite" ]
+then
+    touch /producer/database/database.sqlite
+    php artisan migrate
+fi
+
+# HORIZON
+sh /producer/.horizon/horizon.sh
+
+php /producer/artisan serve --host "0.0.0.0"