You can now write serializers for TL in PHP (support in vkext) (#1416)

Author: hrissan

vkext/vk_zend.h

@@ -226,6 +226,25 @@ static zend_always_inline void vk_zend_update_public_property_long(zval *object,
 #endif
 }
 
+static zend_always_inline void vk_zend_call_known_instance_method(zval *object,
+                                                                    const char * name, size_t name_len, zval *retval_ptr,
+                                                                    uint32_t param_count, zval *params) {
+  zend_object* zobj = Z_OBJ_P(object);
+  zend_string* method_name = zend_string_init(name, name_len, 0);
+  zend_function* fun = Z_OBJ_HANDLER_P(object, get_method)(&zobj, method_name, 0);
+  zend_string_release(method_name);
+  if (!fun) {
+    return; // retval stays UNDEF
+  }
+#if PHP_MAJOR_VERSION >= 8
+  zend_call_known_instance_method(fun, zobj, retval_ptr, param_count, params);
+#else
+  zval method_name_zval;
+  ZVAL_STR(&method_name_zval, method_name);
+  call_user_function(NULL, object, &method_name_zval, retval_ptr, param_count, params);
+#endif
+}
+
 #define ZAPI_TO_PP(az)          (&(az))
 #define ZP_TO_API_P(az)         (az)
 #define SMART_STRDATA(ss)       ((ss).s)

vkext/vkext-rpc-req-error.cpp

@@ -68,13 +68,15 @@ void tl::RpcReqError::tl_fetch() {
 }
 
 void RpcError::try_fetch() {
-  int op = tl_parse_int();
+  int op = tl_lookup_int();
   if (op == TL_REQ_RESULT_HEADER) {
+    (void)tl_parse_int(); // skip op
     flags = tl_parse_int();
     header.emplace().tl_fetch(flags);
-    op = tl_parse_int();
+    op = tl_lookup_int();
   }
   if (op == TL_RPC_REQ_ERROR) {
+    (void)tl_parse_int(); // skip op
     error.emplace().tl_fetch();
   }
 }

vkext/vkext-rpc-tl-serialization.cpp

@@ -532,7 +532,7 @@ static zval *create_php_instance(const char *class_name) {
   return ci;
 }
 
-static zval *make_query_result_or_error(zval **r, const vkext_rpc::tl::RpcReqError &error, const vkext_rpc::tl::RpcReqResultExtra *header = nullptr, int extra_flags = 0);
+static zval *make_query_result_or_error(zval *r, const vkext_rpc::tl::RpcReqError &error, const vkext_rpc::tl::RpcReqResultExtra *header = nullptr, int extra_flags = 0);
 
 /**
  * This function extracts ORIGINAL tl name from given php class name.
@@ -924,6 +924,39 @@ inline void tl_debug(const char *s __attribute__((unused)), int n __attribute__(
 
 /* {{{ Interface functions */
 
+// returns 0 if no typedStore was found, otherwise returns allocated ZVAL with fetcher instance
+bool store_function2(VK_ZVAL_API_P arr, zval *fetcher) {
+  ADD_CNT(store_function2)
+  START_TIMER(store_function2)
+  assert(arr);
+  if (Z_TYPE_P(arr) != IS_OBJECT) {
+    END_TIMER(store_function2)
+    return false;
+  }
+  vk_zend_call_known_instance_method(arr, "typedStore", strlen("typedStore"), fetcher, 0, NULL);
+  if (EG(exception)) {
+    // This behavior is consistent with old code, in this case, query_one will return qid 0
+    fprintf(stderr, "typedStore exception\n");
+    _zend_object* old_exception = EG(exception);
+    EG(exception) = NULL;
+    OBJ_RELEASE(old_exception);
+    END_TIMER(store_function2)
+    return false;
+  }
+  if (Z_TYPE_P(fetcher) != IS_OBJECT) {
+    // returned null or function not found (undef) or function returned something unexpected
+    if (Z_TYPE_P(fetcher) != IS_NULL) {
+      fprintf(stderr, "typedStore fetcher unexpected type is %d\n", Z_TYPE_P(fetcher));
+    }
+    END_TIMER(store_function2)
+    return false;
+  }
+  // when using fetcher, tl_current_function_name will not be accessed. But we set it anyway in case we forgot something.
+  tl_current_function_name = "typedStore";
+  END_TIMER(store_function2)
+  return true;
+}
+
 struct tl_tree *store_function(VK_ZVAL_API_P arr) {
   ADD_CNT(store_function)
   START_TIMER(store_function)
@@ -1034,7 +1067,7 @@ struct tl_tree *store_function(VK_ZVAL_API_P arr) {
   return reinterpret_cast<tl_tree *>(res);
 }
 
-zval **fetch_function(struct tl_tree *T) {
+zval *fetch_function(struct tl_tree *T) {
   ADD_CNT(fetch_function)
   START_TIMER(fetch_function)
 #ifdef VLOG
@@ -1057,10 +1090,10 @@ zval **fetch_function(struct tl_tree *T) {
   vkext_rpc::RpcError rpc_error;
   rpc_error.try_fetch();
   if (rpc_error.error.has_value()) {
-    *_arr = make_query_result_or_error(NULL, rpc_error.error.value(), rpc_error.header.has_value() ? &rpc_error.header.value() : nullptr, rpc_error.flags);
+    zval *ret = make_query_result_or_error(NULL, rpc_error.error.value(), rpc_error.header.has_value() ? &rpc_error.header.value() : nullptr, rpc_error.flags);
     DEC_REF (T);
     END_TIMER(fetch_function)
-    return _arr;
+    return ret;
   }
   tl_parse_restore_pos(pos);
 
@@ -1078,28 +1111,55 @@ zval **fetch_function(struct tl_tree *T) {
       VK_ALLOC_INIT_ZVAL(*_arr);
       ZVAL_BOOL (*_arr, 1);
     }
-    return _arr;
+    return *_arr;
   } else {
     if (*_arr) {
       zval_dtor (*_arr);
     }
     *_arr = make_query_result_or_error(NULL, {TL_ERROR_RESPONSE_SYNTAX, "Can't parse response"});
-    return _arr;
+    return *_arr;
   }
 }
 
 void _extra_dec_ref(struct rpc_query *q) {
-  if (q->extra) {
-    total_tl_working--;
+  if (!q->extra_free) {
+    return;
   }
-  DEC_REF (q->extra);
-  q->extra = 0;
   q->extra_free = 0;
+  total_tl_working--;
+  if (q->extra) {
+    DEC_REF (q->extra);
+    q->extra = 0;
+  }
+  zval_ptr_dtor(&q->fetcher);
 }
 
 struct rpc_query *vk_rpc_tl_query_one_impl(struct rpc_connection *c, double timeout, VK_ZVAL_API_P arr, int ignore_answer) {
   do_rpc_clean();
   START_TIMER (tmp);
+  zval fetcher;
+  ZVAL_NULL(&fetcher);
+  bool fetcher_found = store_function2(arr, &fetcher);
+  END_TIMER (tmp);
+  if (fetcher_found) {
+    struct rpc_query *q;
+    if (!(q = do_rpc_send_noflush(c, timeout, ignore_answer))) {
+      zval_ptr_dtor(&fetcher);
+      vkext_error(VKEXT_ERROR_NETWORK, "Can't send packet");
+      return 0;
+    }
+    if (q == (struct rpc_query *)1) { // answer is ignored
+      assert (ignore_answer);
+      zval_ptr_dtor(&fetcher);
+      return q;
+    }
+    assert (!ignore_answer);
+    ZVAL_COPY_VALUE(&q->fetcher, &fetcher);
+    q->extra_free = _extra_dec_ref;
+    total_tl_working++;
+    return q;
+  }
+  START_TIMER (tmp);
   void *res = store_function(arr);
   END_TIMER (tmp);
   if (!res) {
@@ -1118,15 +1178,23 @@ struct rpc_query *vk_rpc_tl_query_one_impl(struct rpc_connection *c, double time
   }
   assert (!ignore_answer);
   q->extra = res;
+  ZVAL_NULL(&q->fetcher);
   q->extra_free = _extra_dec_ref;
   total_tl_working++;
   return q;
 }
 
-zval **vk_rpc_tl_query_result_one_impl(struct tl_tree *T) {
+zval *fetch_function2(zval *fetcher);
+
+zval *vk_rpc_tl_query_result_one_impl(struct tl_tree *T, zval *fetcher) {
   tl_parse_init();
   START_TIMER (tmp);
-  zval **r = fetch_function(T);
+  zval *r = NULL;
+  if (T) {
+    r = fetch_function(T);
+  }else{
+    r = fetch_function2(fetcher);
+  }
   //fprintf(stderr, "~~~~ after fetch:\n");
   //php_debug_zval_dump(*r, 1);
   END_TIMER (tmp);
@@ -1352,9 +1420,80 @@ static zval *convert_rpc_extra_header_to_php_repr(const vkext_rpc::tl::RpcReqRes
   return res;
 }
 
-static zval *make_query_result_or_error(zval **r, const vkext_rpc::tl::RpcReqError &error, const vkext_rpc::tl::RpcReqResultExtra *header, int extra_flags) {
+zval *fetch_function2(zval *fetcher) {
+  ADD_CNT(fetch_function2)
+  START_TIMER(fetch_function2)
+
+  assert(fetcher);
+  assert(Z_TYPE_P(fetcher) == IS_OBJECT);
+
+  vkext_rpc::RpcError rpc_error;
+  rpc_error.try_fetch();
+  if (rpc_error.error.has_value()) {
+    zval *ret = make_query_result_or_error(NULL, rpc_error.error.value(), rpc_error.header.has_value() ? &rpc_error.header.value() : nullptr, rpc_error.flags);
+    END_TIMER(fetch_function2)
+    return ret;
+  }
+
+  zval* return_value;
+  VK_ALLOC_INIT_ZVAL(return_value);
+  ZVAL_UNDEF(return_value);
+  vk_zend_call_known_instance_method(fetcher, "typedFetch", strlen("typedFetch"), return_value, 0, NULL);
+  if (EG(exception)) {
+    efree(return_value); // it is UNDEF
+
+    _zend_object * old_exception = EG(exception);
+    EG(exception) = NULL;
+
+    zval exception_zval;
+    ZVAL_OBJ(&exception_zval, old_exception);
+
+    zval *message;
+    VK_ALLOC_INIT_ZVAL(message);
+    ZVAL_UNDEF(message);
+    vk_zend_call_known_instance_method(&exception_zval, "getMessage", strlen("getMessage"), message, 0, NULL);
+    // fprintf(stderr, "getMessage after call %d\n", Z_TYPE(message));
+    assert(Z_TYPE_P(message) == IS_STRING);
+
+    OBJ_RELEASE(old_exception);
+
+    zval *_err = create_php_instance(reqResult_error_class_name);
+
+    vk_zend_update_public_property_nod(_err, "error", message);
+
+    // vk_zend_update_public_property_string(_err, "error", "hren");
+    vk_zend_update_public_property_long(_err, "error_code", -1000);
+    END_TIMER(fetch_function2)
+    return _err;
+  }
+  // TODO - will remove later when everything works
+  // fprintf(stderr, "typedFetch after call %d\n", Z_TYPE_P(return_value));
+  if (Z_TYPE_P(return_value) != IS_OBJECT) { // should be never, but that is user code
+    zval *_err = create_php_instance(reqResult_error_class_name);
+    vk_zend_update_public_property_string(_err, "error", "fetcher->typedFetch() did not return object, as expected");
+    vk_zend_update_public_property_long(_err, "error_code", -1000);
+    END_TIMER(fetch_function2)
+    return _err;
+  }
+  if (rpc_error.header.has_value()) {
+    zval *wrapped_err = create_php_instance(reqResult_header_class_name);
+    zval *header_php_repr = convert_rpc_extra_header_to_php_repr(rpc_error.header.value());
+
+    set_field_int(&wrapped_err, rpc_error.flags, "flags", -1);
+    set_field(&wrapped_err, header_php_repr, "extra", -1);
+    set_field(&wrapped_err, return_value, "result", -1);
+    END_TIMER(fetch_function2)
+    return wrapped_err;
+  }
+  zval *wrapped_err = create_php_instance(reqResult_underscore_class_name);
+  set_field(&wrapped_err, return_value, "result", -1);
+  END_TIMER(fetch_function2)
+  return wrapped_err;
+}
+
+static zval *make_query_result_or_error(zval *r, const vkext_rpc::tl::RpcReqError &error, const vkext_rpc::tl::RpcReqResultExtra *header, int extra_flags) {
   if (r) {
-    return *r;
+    return r;
   }
   zval *_err;
   switch (typed_mode) {
@@ -1404,13 +1543,22 @@ void vk_rpc_tl_query_result_impl(struct rpc_queue *Q, double timeout, zval **r)
     }
     struct rpc_query *q = rpc_query_get(qid);
     tl_tree *T = reinterpret_cast<tl_tree *>(q->extra);
+    zval fetcher;
+    ZVAL_COPY(&fetcher, &q->fetcher);
     tl_current_function_name = q->fun_name;
-    INC_REF (T);
+    if (T) {
+      INC_REF (T);
+    }
 
     if (do_rpc_get_and_parse(qid, timeout - precise_now) < 0) {
+      // TODO - most likely. leak here (of both T and fetcher).
+      // But it is difficult to simulate this situation, so
+      // we decided to keep leak to avoid double delete in case we
+      // failed to completely understand this code.
       continue;
     }
-    zval *res = make_query_result_or_error(vk_rpc_tl_query_result_one_impl(T), {TL_ERROR_RESPONSE_NOT_FOUND, "Response not found, probably timed out"});
+    zval *res = make_query_result_or_error(vk_rpc_tl_query_result_one_impl(T, &fetcher), {TL_ERROR_RESPONSE_NOT_FOUND, "Response not found, probably timed out"});
+    zval_ptr_dtor(&fetcher);
     vk_add_index_zval_nod (*r, qid, res);
   }
 }
@@ -1441,14 +1589,19 @@ void vk_rpc_tl_query_result_one(INTERNAL_FUNCTION_PARAMETERS) {
   double timeout = (argc < 2) ? q->timeout : precise_now + parse_zend_double(VK_ZVAL_ARRAY_TO_API_P(z[1]));
   END_TIMER (parse);
   auto *T = reinterpret_cast<tl_tree *>(q->extra);
-  INC_REF (T);
+  zval fetcher;
+  ZVAL_COPY(&fetcher, &q->fetcher);
+  if (T) {
+    INC_REF (T);
+  }
   if (do_rpc_get_and_parse(qid, timeout - precise_now) < 0) {
     zval *r = make_query_result_or_error(NULL, {TL_ERROR_RESPONSE_NOT_FOUND, "Response not found, probably timed out"});
     RETVAL_ZVAL(r, false, true);
     efree(r);
     return;
   }
-  zval *r = make_query_result_or_error(vk_rpc_tl_query_result_one_impl(T), {TL_ERROR_RESPONSE_NOT_FOUND, "Response not found, probably timed out"});
+  zval *r = make_query_result_or_error(vk_rpc_tl_query_result_one_impl(T, &fetcher), {TL_ERROR_RESPONSE_NOT_FOUND, "Response not found, probably timed out"});
+  zval_ptr_dtor(&fetcher);
   RETVAL_ZVAL(r, false, true);
   efree(r);
 }

vkext/vkext-rpc.cpp

@@ -854,9 +854,7 @@ static struct rpc_query *rpc_query_alloc(double timeout) {
   q->qid = qid;
   q->start_time = precise_now;
   q->timeout = timeout;
-  if (tl_current_function_name) {
-    q->fun_name = tl_current_function_name;
-  }
+  q->fun_name = tl_current_function_name;
 /*  ADD_CNT(tree_insert);
   START_TICKS(tree_insert);
   query_tree = tree_insert_query (query_tree, q, lrand48 ());

vkext/vkext-rpc.h

@@ -12,7 +12,7 @@
 #define RPC_BUF_SIZE (1 << 12)
 #define RPC_SERVER_MAGIC 0x8940303d
 #define RPC_BUFFER_MAGIC 0x8fa0da0c
-#define RPC_MAX_QUERY_LEN (1 << 24)
+#define RPC_MAX_QUERY_LEN ((1 << 24) - 1)
 
 #define RPC_SKIP 0
 
@@ -113,6 +113,7 @@ struct rpc_query {
   int answer_len;
   enum query_status status;
   void *extra;
+  zval fetcher; // set if extra == NULL
   void (*extra_free)(struct rpc_query *);
   const char *fun_name;
 };
@@ -303,7 +304,9 @@ struct stats {
   DECLARE_STAT(store);
   DECLARE_STAT(fetch);
   DECLARE_STAT(store_function);
+  DECLARE_STAT(store_function2);
   DECLARE_STAT(fetch_function);
+  DECLARE_STAT(fetch_function2);
   DECLARE_STAT(crc32);
   DECLARE_STAT(tree_insert);
   DECLARE_STAT(total);

vkext/vkext-tl-parse.cpp

@@ -27,6 +27,13 @@ int tl_parse_int() {
   return do_rpc_fetch_int(&tl.error);
 }
 
+int tl_lookup_int() {
+  if (tl.error) {
+    return -1;
+  }
+  return do_rpc_lookup_int(&tl.error);
+}
+
 long long tl_parse_long() {
   if (tl.error) {
     return -1;
@@ -90,6 +97,7 @@ std::string tl_parse_string() {
     return res;
   }
   res.assign(s);
+  free(s);
   return res;
 }