darlinghq/darling-newlkm
1
0
Fork 0
mirror of https://github.com/darlinghq/darling-newlkm.git synced 2024-12-02 09:06:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity
master
darling-newlkm/darling/psynch_support.c
Ariel Abreu de0b2756a7
Partially revert 1cbb1e2457 
The updated port has a few invalid memory access issues and lockups that I don't want to debug right now (I just want to get this entire codebase update done). The old code was working just fine, so let's use that until we decide to update it.
2020-11-20 11:08:29 -05:00

4240 lines
130 KiB
C
Raw Permalink Blame History

// Ported to Darling by Lubos Dolezel
/*
* Copyright (c) 2000-2009 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
* This file contains Original Code and/or Modifications of Original Code
* as defined in and that are subject to the Apple Public Source License
* Version 2.0 (the 'License'). You may not use this file except in
* compliance with the License. The rights granted to you under the License
* may not be used to create, or enable the creation or redistribution of,
* unlawful or unlicensed copies of an Apple operating system, or to
* circumvent, violate, or enable the circumvention or violation of, any
* terms of an Apple operating system software license agreement.
*
* Please obtain a copy of the License at
* http://www.opensource.apple.com/apsl/ and read it before using this file.
*
* The Original Code and all software distributed under the License are
* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
* Please see the License for the specific language governing rights and
* limitations under the License.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*/
/* Copyright (c) 1995-2005 Apple Computer, Inc. All Rights Reserved */
/*
* pthread_support.c
*/
#include <duct/duct.h>
#include <duct/duct_pre_xnu.h>
#include <kern/task.h>
#include <kern/thread_call.h>
#include <kern/kern_types.h>
#include <kern/ipc_tt.h>
#include <sys/proc_internal.h>
#include <sys/user.h>
#include <sys/malloc.h>
#include <duct/duct_kern_printf.h>
#include <duct/duct_post_xnu.h>
#include <linux/list.h>
#include <linux/mutex.h>
#include <linux/slab.h>
#include <stdint.h>
#include <linux/string.h>
#include <linux/bug.h>
#include <linux/jiffies.h>
#include <linux/delay.h>
#include <linux/time.h>
#include "pthread_internal.h"
#include "task_registry.h"
#include "api.h"
#include "pthread_kext.h"
#undef wait_queue_t
#define wait_queue_t struct __wait_queue
#undef kfree
#undef zfree
//#undef panic
#define zfree(where, what) kfree(what)
#define __pthread_testcancel(x) if (darling_thread_canceled()) return LINUX_EINTR;
//#define panic(str, arg...) WARN(1, str)
#define task_threadmax 1024
#define lck_mtx_destroy(lock, prop)
#define unix_syscall_return thread_syscall_return
//typedef kern_return_t wait_result_t;
#define M_PROC 0
#define pthhashhead list_head
#define nanoseconds_to_absolutetime(ns, to) *to = ns
#if LINUX_VERSION_CODE >= KERNEL_VERSION(4,20,0)
#define microuptime(t) ktime_get_ts64(t)
#else
#define microuptime(t) ktime_get_ts(t)
#endif
#define tvtoabstime(t) ((t)->tv_nsec + ((uint64_t)(t)->tv_sec) * NSEC_PER_SEC)
// To shut up a few warnings
struct psynch_rw_longrdlock_args;
struct psynch_rw_yieldwrlock_args;
struct psynch_rw_upgrade_args;
struct psynch_rw_downgrade_args;
struct psynch_rw_unlock2_args;
#define __PSYNCH_DEBUG__ 0 /* debug panic actions */
#if (KDEBUG && STANDARD_KDEBUG)
#define _PSYNCH_TRACE_ 1 /* kdebug trace */
#endif
#define __TESTMODE__ 2 /* 0 - return error on user error conditions */
/* 1 - log error on user error conditions */
/* 2 - abort caller on user error conditions */
/* 3 - panic on user error conditions */
static int __test_panics__;
static int __test_aborts__;
static int __test_prints__;
#define __FAILEDUSERTEST__(str) printf("psynch: " str)
#define ECVCERORR 256
#define ECVPERORR 512
DEFINE_MUTEX(pthread_list_mlock);
#define LIST_ENTRY(where) struct list_head
#define LIST_INIT INIT_LIST_HEAD
#define LIST_HEAD(name) struct list_head name = LIST_HEAD_INIT(name)
#define TAILQ_FIRST(head) (list_empty(head) ? NULL : list_first_entry(head, struct ksyn_waitq_element, kwe_list))
#define TAILQ_LAST(head, member) list_last_entry(head, struct ksyn_waitq_element, kwe_list)
#define TAILQ_FOREACH_SAFE(elem, list, entry, temp) list_for_each_entry_safe(elem, temp, list, entry)
#define TAILQ_REMOVE(list, elem, member) list_del(&(elem)->member)
#define TAILQ_INSERT_HEAD(head, entry, member) list_add(&(entry)->member, head)
#define TAILQ_INSERT_TAIL(head, entry, member) list_add_tail(&(entry)->member, head)
#define TAILQ_INSERT_BEFORE(where, what, member) list_add_tail(&(what)->member, &(where)->member)
#define LIST_FOREACH(entry, head, member) typeof(entry) __foreach_tmp; list_for_each_entry_safe(entry, __foreach_tmp, head, member)
#define LIST_REMOVE(entry, member) list_del(&(entry)->member)
#define LIST_INSERT_HEAD(list, what, member) list_add(&(what)->member, list)
#define LIST_FIRST(head, member) (list_empty(head) ? NULL : list_first_entry(head, struct ksyn_wait_queue, member))
struct list_head* pth_glob_hashtbl;
u_long pthhash;
LIST_HEAD(pth_free_list);
int num_total_kwq = 0; /* number of kwq in use currently */
int num_infreekwq = 0; /* number of kwq in free list */
int num_freekwq = 0; /* number of kwq actually freed from the free the list */
int num_reusekwq = 0; /* number of kwq pulled back for reuse from free list */
int num_addedfreekwq = 0; /* number of added free kwq from the last instance */
int num_lastfreekwqcount = 0; /* the free count from the last time */
static int PTH_HASHSIZE = 100;
//static zone_t kwq_zone; /* zone for allocation of ksyn_queue */
//static zone_t kwe_zone; /* zone for allocation of ksyn_waitq_element */
#define SEQFIT 0
#define FIRSTFIT 1
struct ksyn_queue {
//TAILQ_HEAD(ksynq_kwelist_head, ksyn_waitq_element) ksynq_kwelist;
struct list_head ksynq_kwelist;
uint32_t ksynq_count; /* number of entries in queue */
uint32_t ksynq_firstnum; /* lowest seq in queue */
uint32_t ksynq_lastnum; /* highest seq in queue */
};
typedef struct ksyn_queue * ksyn_queue_t;
#define KSYN_QUEUE_READ 0
#define KSYN_QUEUE_LREAD 1
#define KSYN_QUEUE_WRITER 2
#define KSYN_QUEUE_YWRITER 3
#define KSYN_QUEUE_UPGRADE 4
#define KSYN_QUEUE_MAX 5
struct ksyn_wait_queue {
LIST_ENTRY(ksyn_wait_queue) kw_hash;
LIST_ENTRY(ksyn_wait_queue) kw_list;
user_addr_t kw_addr;
uint64_t kw_owner;
uint64_t kw_object; /* object backing in shared mode */
uint64_t kw_offset; /* offset inside the object in shared mode */
int kw_flags; /* mutex, cvar options/flags */
int kw_pflags; /* flags under listlock protection */
struct linux_timespec kw_ts; /* timeval need for upkeep before free */
int kw_iocount; /* inuse reference */
int kw_dropcount; /* current users unlocking... */
int kw_type; /* queue type like mutex, cvar, etc */
uint32_t kw_inqueue; /* num of waiters held */
uint32_t kw_fakecount; /* number of error/prepost fakes */
uint32_t kw_highseq; /* highest seq in the queue */
uint32_t kw_lowseq; /* lowest seq in the queue */
uint32_t kw_lword; /* L value from userland */
uint32_t kw_uword; /* U world value from userland */
uint32_t kw_sword; /* S word value from userland */
uint32_t kw_lastunlockseq; /* the last seq that unlocked */
/* for CV to be used as the seq kernel has seen so far */
#define kw_cvkernelseq kw_lastunlockseq
uint32_t kw_lastseqword; /* the last seq that unlocked */
/* for mutex and cvar we need to track I bit values */
uint32_t kw_nextseqword; /* the last seq that unlocked; with num of waiters */
#define kw_initrecv kw_nextseqword /* number of incoming waiters with Ibit seen sofar */
uint32_t kw_overlapwatch; /* chance for overlaps */
#define kw_initcount kw_overlapwatch /* number of incoming waiters with Ibit expected */
uint32_t kw_initcountseq; /* highest seq with Ibit on for mutex and cvar*/
uint32_t kw_pre_rwwc; /* prepost count */
uint32_t kw_pre_lockseq; /* prepost target seq */
uint32_t kw_pre_sseq; /* prepost target sword, in cvar used for mutexowned */
uint32_t kw_pre_intrcount; /* prepost of missed wakeup due to intrs */
uint32_t kw_pre_intrseq; /* prepost of missed wakeup limit seq */
uint32_t kw_pre_intrretbits; /* return bits value for missed wakeup threads */
uint32_t kw_pre_intrtype; /* type of failed wakueps*/
int kw_kflags;
struct ksyn_queue kw_ksynqueues[KSYN_QUEUE_MAX]; /* queues to hold threads */
// lck_mtx_t kw_lock; /* mutex lock protecting this structure */
struct mutex kw_lock;
struct __wait_queue_head linux_wq;
};
typedef struct ksyn_wait_queue * ksyn_wait_queue_t;
#define PTHRW_INC 0x100
#define PTHRW_BIT_MASK 0x000000ff
#define PTHRW_COUNT_SHIFT 8
#define PTHRW_COUNT_MASK 0xffffff00
#define PTHRW_MAX_READERS 0xffffff00
/* New model bits on Lword */
#define PTH_RWL_KBIT 0x01 /* users cannot acquire in user mode */
#define PTH_RWL_EBIT 0x02 /* exclusive lock in progress */
#define PTH_RWL_WBIT 0x04 /* write waiters pending in kernel */
#define PTH_RWL_PBIT 0x04 /* prepost (cv) pending in kernel */
#define PTH_RWL_YBIT 0x08 /* yielding write waiters pending in kernel */
#define PTH_RWL_RETRYBIT 0x08 /* mutex retry wait */
#define PTH_RWL_LBIT 0x10 /* long read in progress */
#define PTH_RWL_MTXNONE 0x10 /* indicates the cvwait does not have mutex held */
#define PTH_RWL_UBIT 0x20 /* upgrade request pending */
#define PTH_RWL_MTX_WAIT 0x20 /* in cvar in mutex wait */
#define PTH_RWL_RBIT 0x40 /* reader pending in kernel(not used) */
#define PTH_RWL_MBIT 0x40 /* overlapping grants from kernel */
#define PTH_RWL_TRYLKBIT 0x40 /* trylock attempt (mutex only) */
#define PTH_RWL_IBIT 0x80 /* lcok reset, held untill first succeesful unlock */
/* UBIT values for mutex, cvar */
#define PTH_RWU_SBIT 0x01
#define PTH_RWU_BBIT 0x02
#define PTHRW_RWL_INIT PTH_RWL_IBIT /* reset state on the lock bits (U)*/
/* New model bits on Sword */
#define PTH_RWS_SBIT 0x01 /* kernel transition seq not set yet*/
#define PTH_RWS_IBIT 0x02 /* Sequence is not set on return from kernel */
#define PTH_RWS_CV_CBIT PTH_RWS_SBIT /* kernel has cleared all info w.r.s.t CV */
#define PTH_RWS_CV_PBIT PTH_RWS_IBIT /* kernel has prepost/fake structs only,no waiters */
#define PTH_RWS_CV_MBIT PTH_RWL_MBIT /* to indicate prepost return */
#define PTH_RWS_WSVBIT 0x04 /* save W bit */
#define PTH_RWS_USVBIT 0x08 /* save U bit */
#define PTH_RWS_YSVBIT 0x10 /* save Y bit */
#define PTHRW_RWS_INIT PTH_RWS_SBIT /* reset on the lock bits (U)*/
#define PTHRW_RWS_SAVEMASK (PTH_RWS_WSVBIT|PTH_RWS_USVBIT|PTH_RWS_YSVBIT) /*save bits mask*/
#define PTHRW_SW_Reset_BIT_MASK 0x000000fe /* remove S bit and get rest of the bits */
#define PTHRW_RWS_INIT PTH_RWS_SBIT /* reset on the lock bits (U)*/
#define PTHRW_UN_BIT_MASK 0x000000bf /* remove overlap bit */
#define PTHREAD_MTX_TID_SWITCHING (uint64_t)-1
/* new L word defns */
#define is_rwl_readinuser(x) ((((x) & (PTH_RWL_UBIT | PTH_RWL_KBIT)) == 0)||(((x) & PTH_RWL_LBIT) != 0))
#define is_rwl_ebit_set(x) (((x) & PTH_RWL_EBIT) != 0)
#define is_rwl_lbit_set(x) (((x) & PTH_RWL_LBIT) != 0)
#define is_rwl_readoverlap(x) (((x) & PTH_RWL_MBIT) != 0)
#define is_rw_ubit_set(x) (((x) & PTH_RWL_UBIT) != 0)
/* S word checks */
#define is_rws_setseq(x) (((x) & PTH_RWS_SBIT))
#define is_rws_setunlockinit(x) (((x) & PTH_RWS_IBIT))
/* first contended seq that kernel sees */
#define KW_MTXFIRST_KSEQ 0x200
#define KW_CVFIRST_KSEQ 1
#define KW_RWFIRST_KSEQ 0x200
int is_seqlower(uint32_t x, uint32_t y);
int is_seqlower_eq(uint32_t x, uint32_t y);
int is_seqhigher(uint32_t x, uint32_t y);
int is_seqhigher_eq(uint32_t x, uint32_t y);
int find_diff(uint32_t upto, uint32_t lowest);
static inline int diff_genseq(uint32_t x, uint32_t y) {
if (x > y) {
return(x-y);
} else {
return((PTHRW_MAX_READERS - y) + x + PTHRW_INC);
}
}
#define TID_ZERO (uint64_t)0
/* bits needed in handling the rwlock unlock */
#define PTH_RW_TYPE_READ 0x01
#define PTH_RW_TYPE_LREAD 0x02
#define PTH_RW_TYPE_WRITE 0x04
#define PTH_RW_TYPE_YWRITE 0x08
#define PTH_RW_TYPE_UPGRADE 0x10
#define PTH_RW_TYPE_MASK 0xff
#define PTH_RW_TYPE_SHIFT 8
#define PTH_RWSHFT_TYPE_READ 0x0100
#define PTH_RWSHFT_TYPE_LREAD 0x0200
#define PTH_RWSHFT_TYPE_WRITE 0x0400
#define PTH_RWSHFT_TYPE_YWRITE 0x0800
#define PTH_RWSHFT_TYPE_MASK 0xff00
/*
* Mutex protocol attributes
*/
#define PTHREAD_PRIO_NONE 0
#define PTHREAD_PRIO_INHERIT 1
#define PTHREAD_PRIO_PROTECT 2
#define PTHREAD_PROTOCOL_FLAGS_MASK 0x3
/*
* Mutex type attributes
*/
#define PTHREAD_MUTEX_NORMAL 0
#define PTHREAD_MUTEX_ERRORCHECK 4
#define PTHREAD_MUTEX_RECURSIVE 8
#define PTHREAD_MUTEX_DEFAULT PTHREAD_MUTEX_NORMAL
#define PTHREAD_TYPE_FLAGS_MASK 0xc
/*
* Mutex pshared attributes
*/
#define PTHREAD_PROCESS_SHARED 0x10
#define PTHREAD_PROCESS_PRIVATE 0x20
#define PTHREAD_PSHARED_FLAGS_MASK 0x30
/*
* Mutex policy attributes
*/
#define _PTHREAD_MUTEX_POLICY_NONE 0
#define _PTHREAD_MUTEX_POLICY_FAIRSHARE 0x040 /* 1 */
#define _PTHREAD_MUTEX_POLICY_FIRSTFIT 0x080 /* 2 */
#define _PTHREAD_MUTEX_POLICY_REALTIME 0x0c0 /* 3 */
#define _PTHREAD_MUTEX_POLICY_ADAPTIVE 0x100 /* 4 */
#define _PTHREAD_MUTEX_POLICY_PRIPROTECT 0x140 /* 5 */
#define _PTHREAD_MUTEX_POLICY_PRIINHERIT 0x180 /* 6 */
#define PTHREAD_POLICY_FLAGS_MASK 0x1c0
#define _PTHREAD_MTX_OPT_HOLDLOCK 0x200
#define _PTHREAD_MTX_OPT_NOMTX 0x400
#define _PTHREAD_MTX_OPT_NOTIFY 0x1000
#define _PTHREAD_MTX_OPT_MUTEX 0x2000 /* this is a mutex type */
#define _PTHREAD_RWLOCK_UPGRADE_TRY 0x10000
/* pflags */
#define KSYN_WQ_INLIST 1
#define KSYN_WQ_INHASH 2
#define KSYN_WQ_SHARED 4
#define KSYN_WQ_WAITING 8 /* threads waiting for this wq to be available */
#define KSYN_WQ_FLIST 0X10 /* in free list to be freed after a short delay */
/* kflags */
#define KSYN_KWF_INITCLEARED 1 /* the init status found and preposts cleared */
#define KSYN_KWF_ZEROEDOUT 2 /* the lword, etc are inited to 0 */
#define KSYN_CLEANUP_DEADLINE 10
int psynch_cleanupset;
thread_call_t psynch_thcall;
#define KSYN_WQTYPE_INWAIT 0x1000
#define KSYN_WQTYPE_INDROP 0x2000
#define KSYN_WQTYPE_MTX 0x1
#define KSYN_WQTYPE_CVAR 0x2
#define KSYN_WQTYPE_RWLOCK 0x4
#define KSYN_WQTYPE_SEMA 0x8
#define KSYN_WQTYPE_BARR 0x10
#define KSYN_WQTYPE_MASK 0x00ff
#define KSYN_MTX_MAX 0x0fffffff
#define KSYN_WQTYPE_MUTEXDROP (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_MTX)
#define KW_UNLOCK_PREPOST 0x01
#define KW_UNLOCK_PREPOST_UPGRADE 0x02
#define KW_UNLOCK_PREPOST_DOWNGRADE 0x04
#define KW_UNLOCK_PREPOST_READLOCK 0x08
#define KW_UNLOCK_PREPOST_LREADLOCK 0x10
#define KW_UNLOCK_PREPOST_WRLOCK 0x20
#define KW_UNLOCK_PREPOST_YWRLOCK 0x40
#define CLEAR_PREPOST_BITS(kwq) {\
kwq->kw_pre_lockseq = 0; \
kwq->kw_pre_sseq = PTHRW_RWS_INIT; \
kwq->kw_pre_rwwc = 0; \
}
#define CLEAR_INITCOUNT_BITS(kwq) {\
kwq->kw_initcount = 0; \
kwq->kw_initrecv = 0; \
kwq->kw_initcountseq = 0; \
}
#define CLEAR_INTR_PREPOST_BITS(kwq) {\
kwq->kw_pre_intrcount = 0; \
kwq->kw_pre_intrseq = 0; \
kwq->kw_pre_intrretbits = 0; \
kwq->kw_pre_intrtype = 0; \
}
#define CLEAR_REINIT_BITS(kwq) {\
if ((kwq->kw_type & KSYN_WQTYPE_MASK) == KSYN_WQTYPE_CVAR) { \
if((kwq->kw_inqueue != 0) && (kwq->kw_inqueue != kwq->kw_fakecount)) \
panic("CV:entries in queue durinmg reinit %d:%d\n",kwq->kw_inqueue, kwq->kw_fakecount); \
};\
if ((kwq->kw_type & KSYN_WQTYPE_MASK) == KSYN_WQTYPE_RWLOCK) { \
kwq->kw_nextseqword = PTHRW_RWS_INIT; \
kwq->kw_overlapwatch = 0; \
}; \
kwq->kw_pre_lockseq = 0; \
kwq->kw_pre_rwwc = 0; \
kwq->kw_pre_sseq = PTHRW_RWS_INIT; \
kwq->kw_lastunlockseq = PTHRW_RWL_INIT; \
kwq->kw_lastseqword = PTHRW_RWS_INIT; \
kwq->kw_pre_intrcount = 0; \
kwq->kw_pre_intrseq = 0; \
kwq->kw_pre_intrretbits = 0; \
kwq->kw_pre_intrtype = 0; \
kwq->kw_lword = 0; \
kwq->kw_uword = 0; \
kwq->kw_sword = PTHRW_RWS_INIT; \
}
void pthread_list_lock(void);
void pthread_list_unlock(void);
void pthread_list_lock_spin(void);
void pthread_list_lock_convert_spin(void);
void ksyn_wqlock(ksyn_wait_queue_t kwq);
void ksyn_wqunlock(ksyn_wait_queue_t kwq);
ksyn_wait_queue_t ksyn_wq_hash_lookup(user_addr_t mutex, proc_t p, int flags, uint64_t object, uint64_t offset);
int ksyn_wqfind(user_addr_t mutex, uint32_t mgen, uint32_t ugen, uint32_t rw_wc, uint64_t tid, int flags, int wqtype , ksyn_wait_queue_t * wq);
void ksyn_wqrelease(ksyn_wait_queue_t mkwq, ksyn_wait_queue_t ckwq, int qfreenow, int wqtype);
extern int ksyn_findobj(uint64_t mutex, uint64_t * object, uint64_t * offset);
static void UPDATE_CVKWQ(ksyn_wait_queue_t kwq, uint32_t mgen, uint32_t ugen, uint32_t rw_wc, uint64_t tid, int wqtype);
kern_return_t ksyn_block_thread_locked(ksyn_wait_queue_t kwq, uint64_t abstime, ksyn_waitq_element_t kwe, int log, thread_continue_t, void * parameter);
kern_return_t ksyn_wakeup_thread(ksyn_wait_queue_t kwq, ksyn_waitq_element_t kwe);
void ksyn_freeallkwe(ksyn_queue_t kq);
uint32_t psynch_mutexdrop_internal(ksyn_wait_queue_t kwq, uint32_t lkseq, uint32_t ugen, int flags);
int kwq_handle_unlock(ksyn_wait_queue_t, uint32_t mgen, uint32_t rw_wc, uint32_t * updatep, int flags, int *blockp, uint32_t premgen);
void ksyn_queue_init(ksyn_queue_t kq);
int ksyn_queue_insert(ksyn_wait_queue_t kwq, ksyn_queue_t kq, uint32_t mgen, uthread_t uth, ksyn_waitq_element_t kwe, int firstfit);
ksyn_waitq_element_t ksyn_queue_removefirst(ksyn_queue_t kq, ksyn_wait_queue_t kwq);
void ksyn_queue_removeitem(ksyn_wait_queue_t kwq, ksyn_queue_t kq, ksyn_waitq_element_t kwe);
int ksyn_queue_move_tofree(ksyn_wait_queue_t kwq, ksyn_queue_t kq, uint32_t upto, ksyn_queue_t freeq, int all, int reease);
void update_low_high(ksyn_wait_queue_t kwq, uint32_t lockseq);
uint32_t find_nextlowseq(ksyn_wait_queue_t kwq);
uint32_t find_nexthighseq(ksyn_wait_queue_t kwq);
int find_seq_till(ksyn_wait_queue_t kwq, uint32_t upto, uint32_t nwaiters, uint32_t *countp);
uint32_t ksyn_queue_count_tolowest(ksyn_queue_t kq, uint32_t upto);
ksyn_waitq_element_t ksyn_queue_find_cvpreposeq(ksyn_queue_t kq, uint32_t cgen);
uint32_t ksyn_queue_cvcount_entries(ksyn_queue_t kq, uint32_t upto, uint32_t from, int * numwaitersp, int * numintrp, int * numprepop);
void ksyn_handle_cvbroad(ksyn_wait_queue_t ckwq, uint32_t upto, uint32_t *updatep);
void ksyn_cvupdate_fixup(ksyn_wait_queue_t ckwq, uint32_t *updatep, ksyn_queue_t kfreeq, int release);
ksyn_waitq_element_t ksyn_queue_find_signalseq(ksyn_wait_queue_t kwq, ksyn_queue_t kq, uint32_t toseq, uint32_t lockseq);
ksyn_waitq_element_t ksyn_queue_find_threadseq(ksyn_wait_queue_t ckwq, ksyn_queue_t kq, thread_t th, uint32_t toseq);
void psynch_cvcontinue(void *, wait_result_t);
void psynch_mtxcontinue(void *, wait_result_t);
int ksyn_wakeupreaders(ksyn_wait_queue_t kwq, uint32_t limitread, int longreadset, int allreaders, uint32_t updatebits, int * wokenp);
int kwq_find_rw_lowest(ksyn_wait_queue_t kwq, int flags, uint32_t premgen, int * type, uint32_t lowest[]);
ksyn_waitq_element_t ksyn_queue_find_seq(ksyn_wait_queue_t kwq, ksyn_queue_t kq, uint32_t seq, int remove);
int kwq_handle_overlap(ksyn_wait_queue_t kwq, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, uint32_t *updatebitsp, int flags , int * blockp);
int kwq_handle_downgrade(ksyn_wait_queue_t kwq, uint32_t mgen, int flags, uint32_t premgen, int * blockp);
void psynch_wq_cleanup(__unused void * param, __unused void * param1);
static void
UPDATE_CVKWQ(ksyn_wait_queue_t kwq, uint32_t mgen, uint32_t ugen, uint32_t rw_wc, __unused uint64_t tid, __unused int wqtype)
{
if ((kwq->kw_type & KSYN_WQTYPE_MASK) == KSYN_WQTYPE_CVAR) {
if ((kwq->kw_kflags & KSYN_KWF_ZEROEDOUT) != 0) {
/* the values of L,U and S are cleared out due to L==S in previous transition */
kwq->kw_lword = mgen;
kwq->kw_uword = ugen;
kwq->kw_sword = rw_wc;
kwq->kw_kflags &= ~KSYN_KWF_ZEROEDOUT;
}
if (is_seqhigher((mgen & PTHRW_COUNT_MASK), (kwq->kw_lword & PTHRW_COUNT_MASK)) != 0)
kwq->kw_lword = mgen;
if (is_seqhigher((ugen & PTHRW_COUNT_MASK), (kwq->kw_uword & PTHRW_COUNT_MASK)) != 0)
kwq->kw_uword = ugen;
if ((rw_wc & PTH_RWS_CV_CBIT) != 0) {
if(is_seqlower(kwq->kw_cvkernelseq, (rw_wc & PTHRW_COUNT_MASK)) != 0) {
kwq->kw_cvkernelseq = (rw_wc & PTHRW_COUNT_MASK);
}
if (is_seqhigher((rw_wc & PTHRW_COUNT_MASK), (kwq->kw_sword & PTHRW_COUNT_MASK)) != 0)
kwq->kw_sword = rw_wc;
}
}
}
/* to protect the hashes, iocounts, freelist */
void
pthread_list_lock(void)
{
mutex_lock(&pthread_list_mlock);
}
void
pthread_list_lock_spin(void)
{
mutex_lock(&pthread_list_mlock);
}
void
pthread_list_lock_convert_spin(void)
{
// UNUSED!
}
void
pthread_list_unlock(void)
{
mutex_unlock(&pthread_list_mlock);
}
/* to protect the indiv queue */
void
ksyn_wqlock(ksyn_wait_queue_t kwq)
{
mutex_lock(&kwq->kw_lock);
}
void
ksyn_wqunlock(ksyn_wait_queue_t kwq)
{
mutex_unlock(&kwq->kw_lock);
}
/* routine to drop the mutex unlocks , used both for mutexunlock system call and drop during cond wait */
uint32_t
psynch_mutexdrop_internal(ksyn_wait_queue_t kwq, uint32_t lkseq, uint32_t ugen, int flags)
{
uint32_t nextgen, low_writer, updatebits, returnbits = 0;
int firstfit = flags & _PTHREAD_MUTEX_POLICY_FIRSTFIT;
ksyn_waitq_element_t kwe = NULL;
kern_return_t kret = KERN_SUCCESS;
nextgen = (ugen + PTHRW_INC);
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_START, (uint32_t)kwq->kw_addr, lkseq, ugen, flags, 0);
#endif /* _PSYNCH_TRACE_ */
ksyn_wqlock(kwq);
redrive:
if (kwq->kw_inqueue != 0) {
updatebits = (kwq->kw_highseq & PTHRW_COUNT_MASK) | (PTH_RWL_EBIT | PTH_RWL_KBIT);
kwq->kw_lastunlockseq = (ugen & PTHRW_COUNT_MASK);
if (firstfit != 0)
{
/* first fit , pick any one */
kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], kwq);
kwe->kwe_psynchretval = updatebits;
kwe->kwe_kwqqueue = NULL;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xcafecaf1, (uint32_t)(thread_tid((struct thread *)(((struct uthread *)(kwe->kwe_uth))->uu_context.vc_thread))), kwe->kwe_psynchretval, 0);
#endif /* _PSYNCH_TRACE_ */
kret = ksyn_wakeup_thread(kwq, kwe);
#if __TESTPANICS__
if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
panic("psynch_mutexdrop_internal: panic unable to wakeup firstfit mutex thread\n");
#endif /* __TESTPANICS__ */
if (kret == KERN_NOT_WAITING)
goto redrive;
} else {
/* handle fairshare */
low_writer = kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_firstnum;
low_writer &= PTHRW_COUNT_MASK;
if (low_writer == nextgen) {
/* next seq to be granted found */
kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], kwq);
/* since the grant could be cv, make sure mutex wait is set incase the thread interrupted out */
kwe->kwe_psynchretval = updatebits | PTH_RWL_MTX_WAIT;
kwe->kwe_kwqqueue = NULL;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xcafecaf2, (uint32_t)(thread_tid((struct thread *)(((struct uthread *)(kwe->kwe_uth))->uu_context.vc_thread))), kwe->kwe_psynchretval, 0);
#endif /* _PSYNCH_TRACE_ */
kret = ksyn_wakeup_thread(kwq, kwe);
#if __TESTPANICS__
if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
panic("psynch_mutexdrop_internal: panic unable to wakeup fairshare mutex thread\n");
#endif /* __TESTPANICS__ */
if (kret == KERN_NOT_WAITING) {
/* interrupt post */
kwq->kw_pre_intrcount = 1;
kwq->kw_pre_intrseq = nextgen;
kwq->kw_pre_intrretbits = updatebits;
kwq->kw_pre_intrtype = PTH_RW_TYPE_WRITE;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xfafafaf1, nextgen, kwq->kw_pre_intrretbits, 0);
#endif /* _PSYNCH_TRACE_ */
}
} else if (is_seqhigher(low_writer, nextgen) != 0) {
kwq->kw_pre_rwwc++;
if (kwq->kw_pre_rwwc > 1) {
__FAILEDUSERTEST__("psynch_mutexdrop_internal: prepost more than one (1)\n");
goto out;
}
kwq->kw_pre_lockseq = (nextgen & PTHRW_COUNT_MASK);
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xfefefef1, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
#endif /* _PSYNCH_TRACE_ */
} else {
//__FAILEDUSERTEST__("psynch_mutexdrop_internal: FS mutex unlock sequence higher than the lowest one is queue\n");
kwe = ksyn_queue_find_seq(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], (nextgen & PTHRW_COUNT_MASK), 1);
if (kwe != NULL) {
/* next seq to be granted found */
/* since the grant could be cv, make sure mutex wait is set incase the thread interrupted out */
kwe->kwe_psynchretval = updatebits | PTH_RWL_MTX_WAIT;
kwe->kwe_kwqqueue = NULL;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xcafecaf3, (uint32_t)(thread_tid((struct thread *)(((struct uthread *)(kwe->kwe_uth))->uu_context.vc_thread))), kwe->kwe_psynchretval, 0);
#endif /* _PSYNCH_TRACE_ */
kret = ksyn_wakeup_thread(kwq, kwe);
#if __TESTPANICS__
if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
panic("psynch_mutexdrop_internal: panic unable to wakeup fairshare mutex thread\n");
#endif /* __TESTPANICS__ */
if (kret == KERN_NOT_WAITING)
goto redrive;
} else {
/* next seq to be granted not found, prepost */
kwq->kw_pre_rwwc++;
if (kwq->kw_pre_rwwc > 1) {
__FAILEDUSERTEST__("psynch_mutexdrop_internal: prepost more than one (2)\n");
goto out;
}
kwq->kw_pre_lockseq = (nextgen & PTHRW_COUNT_MASK);
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xfefefef2, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
#endif /* _PSYNCH_TRACE_ */
}
}
}
} else {
/* if firstfit the last one could be spurious */
if (firstfit == 0) {
kwq->kw_lastunlockseq = (ugen & PTHRW_COUNT_MASK);
kwq->kw_pre_rwwc++;
if (kwq->kw_pre_rwwc > 1) {
__FAILEDUSERTEST__("psynch_mutexdrop_internal: prepost more than one (3)\n");
goto out;
}
kwq->kw_pre_lockseq = (nextgen & PTHRW_COUNT_MASK);
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xfefefef3, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
#endif /* _PSYNCH_TRACE_ */
} else {
/* first fit case */
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xfefefef3, kwq->kw_lastunlockseq, kwq->kw_pre_lockseq, 0);
#endif /* _PSYNCH_TRACE_ */
kwq->kw_lastunlockseq = (ugen & PTHRW_COUNT_MASK);
/* not set or the new lkseq is higher */
if ((kwq->kw_pre_rwwc == 0) || (is_seqlower(kwq->kw_pre_lockseq, lkseq) == 0))
kwq->kw_pre_lockseq = (lkseq & PTHRW_COUNT_MASK);
kwq->kw_pre_rwwc = 1;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xfefefef3, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
#endif /* _PSYNCH_TRACE_ */
/* indicate prepost content in kernel */
returnbits = lkseq | PTH_RWL_PBIT;
}
}
out:
ksyn_wqunlock(kwq);
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_KMDROP | DBG_FUNC_END, (uint32_t)kwq->kw_addr, 0xeeeeeeed, 0, 0, 0);
#endif /* _PSYNCH_TRACE_ */
ksyn_wqrelease(kwq, NULL, 1, (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_MTX));
return(returnbits);
}
/*
* psynch_mutexwait: This system call is used for contended psynch mutexes to block.
*/
int
darling_psynch_mutexwait(__unused proc_t p, user_addr_t mutex, uint32_t mgen, uint32_t ugen, uint64_t tid, uint32_t flags, uint32_t* retval)
{
ksyn_wait_queue_t kwq;
int error=0;
int ins_flags, retry;
uthread_t uth;
int firstfit = flags & _PTHREAD_MUTEX_POLICY_FIRSTFIT;
uint32_t lockseq, updatebits=0;
ksyn_waitq_element_t kwe;
kern_return_t kret;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_START, (uint32_t)mutex, mgen, ugen, flags, 0);
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_NONE, (uint32_t)mutex, mgen, ugen, (uint32_t)tid, 0);
#endif /* _PSYNCH_TRACE_ */
uth = current_uthread();
kwe = &uth->uu_save.uus_kwe;
sema_init(&kwe->linux_sem, 0);
kwe->kwe_lockseq = mgen;
kwe->kwe_uth = uth;
kwe->kwe_psynchretval = 0;
kwe->kwe_kwqqueue = NULL;
lockseq = (mgen & PTHRW_COUNT_MASK);
if (firstfit == 0) {
ins_flags = SEQFIT;
} else {
/* first fit */
ins_flags = FIRSTFIT;
}
error = ksyn_wqfind(mutex, mgen, ugen, 0, tid, flags, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_MTX), &kwq);
if (error != 0) {
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_END, (uint32_t)mutex, 1, 0xdeadbeef, error, 0);
#endif /* _PSYNCH_TRACE_ */
return(error);
}
ksyn_wqlock(kwq);
if ((mgen & PTH_RWL_RETRYBIT) != 0) {
retry = 1;
mgen &= ~PTH_RWL_RETRYBIT;
}
/* handle first the missed wakeups */
if ((kwq->kw_pre_intrcount != 0) &&
((kwq->kw_pre_intrtype == PTH_RW_TYPE_WRITE)) &&
(is_seqlower_eq(lockseq, (kwq->kw_pre_intrseq & PTHRW_COUNT_MASK)) != 0)) {
kwq->kw_pre_intrcount--;
kwe->kwe_psynchretval = kwq->kw_pre_intrretbits;
if (kwq->kw_pre_intrcount==0)
CLEAR_INTR_PREPOST_BITS(kwq);
ksyn_wqunlock(kwq);
*retval = kwe->kwe_psynchretval;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_NONE, (uint32_t)mutex, 0xfafafaf1, kwe->kwe_psynchretval, kwq->kw_pre_intrcount, 0);
#endif /* _PSYNCH_TRACE_ */
goto out;
}
if ((kwq->kw_pre_rwwc != 0) && ((ins_flags == FIRSTFIT) || ((lockseq & PTHRW_COUNT_MASK) == (kwq->kw_pre_lockseq & PTHRW_COUNT_MASK) ))) {
/* got preposted lock */
kwq->kw_pre_rwwc--;
if (kwq->kw_pre_rwwc == 0) {
CLEAR_PREPOST_BITS(kwq);
kwq->kw_lastunlockseq = PTHRW_RWL_INIT;
if (kwq->kw_inqueue == 0) {
updatebits = lockseq | (PTH_RWL_KBIT | PTH_RWL_EBIT);
} else {
updatebits = (kwq->kw_highseq & PTHRW_COUNT_MASK) | (PTH_RWL_KBIT | PTH_RWL_EBIT);
}
updatebits &= ~PTH_RWL_MTX_WAIT;
kwe->kwe_psynchretval = updatebits;
if (updatebits == 0) {
__FAILEDUSERTEST__("psynch_mutexwait(prepost): returning 0 lseq in mutexwait with no EBIT \n");
}
ksyn_wqunlock(kwq);
*retval = updatebits;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xfefefef1, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
#endif /* _PSYNCH_TRACE_ */
goto out;
} else {
__FAILEDUSERTEST__("psynch_mutexwait: more than one prepost\n");
kwq->kw_pre_lockseq += PTHRW_INC; /* look for next one */
ksyn_wqunlock(kwq);
error = LINUX_EINVAL;
goto out;
}
}
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 0xfeedfeed, mgen, ins_flags, 0);
#endif /* _PSYNCH_TRACE_ */
error = ksyn_queue_insert(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], mgen, uth, kwe, ins_flags);
if (error != 0) {
ksyn_wqunlock(kwq);
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_END, (uint32_t)mutex, 2, 0xdeadbeef, error, 0);
#endif /* _PSYNCH_TRACE_ */
goto out;
}
kret = ksyn_block_thread_locked(kwq, (uint64_t)0, kwe, 0, psynch_mtxcontinue, (void *)kwq);
psynch_mtxcontinue((void *)kwq, kret);
/* not expected to return from unix_syscall_return */
panic("psynch_mtxcontinue returned from unix_syscall_return");
out:
ksyn_wqrelease(kwq, NULL, 1, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_MTX));
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_END, (uint32_t)mutex, 0xeeeeeeed, updatebits, error, 0);
#endif /* _PSYNCH_TRACE_ */
return(error);
}
void
psynch_mtxcontinue(void * parameter, wait_result_t result)
{
int error = 0;
uint32_t updatebits = 0;
uthread_t uth = current_uthread();
ksyn_wait_queue_t kwq = (ksyn_wait_queue_t)parameter;
ksyn_waitq_element_t kwe;
kwe = &uth->uu_save.uus_kwe;
switch (result) {
case THREAD_TIMED_OUT:
error = LINUX_ETIMEDOUT;
break;
case THREAD_INTERRUPTED:
error = LINUX_EINTR;
break;
default:
error = 0;
break;
}
if (error != 0) {
ksyn_wqlock(kwq);
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 3, 0xdeadbeef, error, 0);
#endif /* _PSYNCH_TRACE_ */
if (kwe->kwe_kwqqueue != NULL)
ksyn_queue_removeitem(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], kwe);
ksyn_wqunlock(kwq);
} else {
updatebits = kwe->kwe_psynchretval;
updatebits &= ~PTH_RWL_MTX_WAIT;
uth->uu_rval[0] = updatebits;
if (updatebits == 0)
__FAILEDUSERTEST__("psynch_mutexwait: returning 0 lseq in mutexwait with no EBIT \n");
}
ksyn_wqrelease(kwq, NULL, 1, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_MTX));
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_MLWAIT | DBG_FUNC_END, (uint32_t)kwq->kw_addr, 0xeeeeeeed, updatebits, error, 0);
#endif /* _PSYNCH_TRACE_ */
unix_syscall_return(error);
}
/*
* psynch_mutexdrop: This system call is used for unlock postings on contended psynch mutexes.
*/
int
darling_psynch_mutexdrop(__unused proc_t p, user_addr_t mutex, uint32_t mgen, uint32_t ugen, uint64_t tid, uint32_t flags, uint32_t* retval)
{
ksyn_wait_queue_t kwq;
uint32_t updateval;
int error=0;
error = ksyn_wqfind(mutex, mgen, ugen, 0, tid, flags, (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_MTX), &kwq);
if (error != 0) {
return(error);
}
updateval = psynch_mutexdrop_internal(kwq, mgen, ugen, flags);
/* drops the kwq reference */
*retval = updateval;
return(0);
}
/*
* psynch_cvbroad: This system call is used for broadcast posting on blocked waiters of psynch cvars.
*/
int
darling_psynch_cvbroad(__unused proc_t p, user_addr_t cond, uint64_t cvlsgen, uint64_t cvudgen, uint32_t flags, user_addr_t mutex, uint64_t mugen, uint64_t tid, uint32_t* retval)
{
uint32_t cgen, cugen, csgen, diffgen;
uint32_t uptoseq, fromseq;
ksyn_wait_queue_t ckwq;
int error=0;
uint32_t updatebits = 0;
uint32_t count;
struct ksyn_queue kfreeq;
csgen = (uint32_t)((cvlsgen >> 32) & 0xffffffff);
cgen = ((uint32_t)(cvlsgen & 0xffffffff));
cugen = (uint32_t)((cvudgen >> 32) & 0xffffffff);
diffgen = ((uint32_t)(cvudgen & 0xffffffff));
count = (diffgen >> PTHRW_COUNT_SHIFT);
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVBROAD | DBG_FUNC_START, (uint32_t)cond, cgen, cugen, csgen, 0);
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVBROAD | DBG_FUNC_NONE, (uint32_t)cond, 0xcbcbcbc1, diffgen,flags, 0);
#endif /* _PSYNCH_TRACE_ */
uptoseq = cgen & PTHRW_COUNT_MASK;
fromseq = (cugen & PTHRW_COUNT_MASK) + PTHRW_INC;
if (is_seqhigher(fromseq, uptoseq) || is_seqhigher((csgen & PTHRW_COUNT_MASK), uptoseq)) {
__FAILEDUSERTEST__("cvbroad: invalid L, U and S values\n");
return LINUX_EINVAL;
}
if (count > (uint32_t)task_threadmax) {
__FAILEDUSERTEST__("cvbroad: difference greater than maximum possible thread count\n");
return LINUX_EBUSY;
}
ckwq = NULL;
error = ksyn_wqfind(cond, cgen, cugen, csgen, 0, flags, (KSYN_WQTYPE_CVAR | KSYN_WQTYPE_INDROP), &ckwq);
if (error != 0) {
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVBROAD | DBG_FUNC_END, (uint32_t)cond, 0, 0xdeadbeef, error, 0);
#endif /* _PSYNCH_TRACE_ */
return(error);
}
*retval = 0;
ksyn_wqlock(ckwq);
/* update L, U and S... */
UPDATE_CVKWQ(ckwq, cgen, cugen, csgen, 0, KSYN_WQTYPE_CVAR);
/* broadcast wakeups/prepost handling */
ksyn_handle_cvbroad(ckwq, uptoseq, &updatebits);
/* set C or P bits and free if needed */
ckwq->kw_sword += (updatebits & PTHRW_COUNT_MASK);
ksyn_cvupdate_fixup(ckwq, &updatebits, &kfreeq, 1);
ksyn_wqunlock(ckwq);
*retval = updatebits;
ksyn_wqrelease(ckwq, NULL, 1, (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_CVAR));
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVBROAD | DBG_FUNC_END, (uint32_t)cond, 0xeeeeeeed, (uint32_t)*retval, error, 0);
#endif /* _PSYNCH_TRACE_ */
return(error);
}
ksyn_waitq_element_t
ksyn_queue_find_threadseq(ksyn_wait_queue_t ckwq, __unused ksyn_queue_t kq, thread_t th, uint32_t upto)
{
uthread_t uth = get_bsdthread_info(th);
ksyn_waitq_element_t kwe = &uth->uu_save.uus_kwe;
if (kwe->kwe_kwqqueue != ckwq ||
is_seqhigher((kwe->kwe_lockseq & PTHRW_COUNT_MASK), upto)) {
/* the thread is not waiting in the cv (or wasn't when the wakeup happened) */
return NULL;
}
return kwe;
}
/*
* psynch_cvsignal: This system call is used for signalling the blocked waiters of psynch cvars.
*/
int
darling_psynch_cvsignal(__unused proc_t p, user_addr_t cond, uint64_t cvlsgen, uint32_t cugen, int threadport, user_addr_t mutex, uint64_t mugen, uint64_t tid, uint32_t flags, uint32_t* retval)
{
uint32_t cgen, csgen, signalseq, uptoseq;
ksyn_wait_queue_t ckwq = NULL;
ksyn_waitq_element_t kwe, nkwe = NULL;
ksyn_queue_t kq;
int error=0;
thread_t th = THREAD_NULL;
uint32_t updatebits = 0;
kern_return_t kret;
struct ksyn_queue kfreeq;
csgen = (uint32_t)((cvlsgen >> 32) & 0xffffffff);
cgen = ((uint32_t)(cvlsgen & 0xffffffff));
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSIGNAL | DBG_FUNC_START, (uint32_t)cond, cgen, cugen, threadport, 0);
#endif /* _PSYNCH_TRACE_ */
uptoseq = cgen & PTHRW_COUNT_MASK;
signalseq = (cugen & PTHRW_COUNT_MASK) + PTHRW_INC;
/* validate sane L, U, and S values */
if (((threadport == 0) && (is_seqhigher(signalseq, uptoseq))) || is_seqhigher((csgen & PTHRW_COUNT_MASK), uptoseq)) {
__FAILEDUSERTEST__("psync_cvsignal; invalid sequence numbers\n");
error = LINUX_EINVAL;
goto out;
}
/* If we are looking for a specific thread, grab a reference for it */
if (threadport != 0) {
th = (thread_t)port_name_to_thread((mach_port_name_t)threadport, PORT_TO_THREAD_NONE);
if (th == THREAD_NULL) {
error = LINUX_ESRCH;
goto out;
}
}
error = ksyn_wqfind(cond, cgen, cugen, csgen, 0, flags, (KSYN_WQTYPE_CVAR | KSYN_WQTYPE_INDROP), &ckwq);
if (error != 0) {
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSIGNAL | DBG_FUNC_END, (uint32_t)cond, 0, 0xdeadbeef, error, 0);
#endif /* _PSYNCH_TRACE_ */
goto out;
}
ksyn_wqlock(ckwq);
/* update L, U and S... */
UPDATE_CVKWQ(ckwq, cgen, cugen, csgen, 0, KSYN_WQTYPE_CVAR);
kq = &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER];
retry:
/* Only bother if we aren't already balanced */
if ((ckwq->kw_lword & PTHRW_COUNT_MASK) != (ckwq->kw_sword & PTHRW_COUNT_MASK)) {
kwe = (th != NULL) ? ksyn_queue_find_threadseq(ckwq, kq, th, uptoseq) :
ksyn_queue_find_signalseq(ckwq, kq, uptoseq, signalseq);
if (kwe != NULL) {
switch (kwe->kwe_flags) {
case KWE_THREAD_BROADCAST:
/* broadcasts swallow our signal */
break;
case KWE_THREAD_PREPOST:
/* merge in with existing prepost at our same uptoseq */
kwe->kwe_count += 1;
break;
case KWE_THREAD_INWAIT:
if (is_seqlower((kwe->kwe_lockseq & PTHRW_COUNT_MASK), signalseq)) {
/*
* A valid thread in our range, but lower than our signal.
* Matching it may leave our match with nobody to wake it if/when
* it arrives (the signal originally meant for this thread might
* not successfully wake it).
*
* Convert to broadcast - may cause some spurious wakeups
* (allowed by spec), but avoids starvation (better choice).
*/
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSIGNAL | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xc1c1c1c1, uptoseq, 0, 0);
#endif /* _PSYNCH_TRACE_ */
ksyn_handle_cvbroad(ckwq, uptoseq, &updatebits);
} else {
ksyn_queue_removeitem(ckwq, kq, kwe);
kwe->kwe_psynchretval = PTH_RWL_MTX_WAIT;
kwe->kwe_kwqqueue = NULL;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSIGNAL | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xcafecaf2, (uint32_t)(thread_tid((struct thread *)(((struct uthread *)(kwe->kwe_uth))->uu_context.vc_thread))), kwe->kwe_psynchretval, 0);
#endif /* _PSYNCH_TRACE_ */
kret = ksyn_wakeup_thread(ckwq, kwe);
#if __TESTPANICS__
if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
panic("ksyn_wakeup_thread: panic waking up condition waiter\n");
#endif /* __TESTPANICS__ */
updatebits += PTHRW_INC;
}
ckwq->kw_sword += (updatebits & PTHRW_COUNT_MASK);
break;
default:
panic("unknown kweflags\n");
break;
}
} else if (th != NULL) {
/*
* Could not find the thread, post a broadcast,
* otherwise the waiter will be stuck. Use to send
* ESRCH here, did lead to rare hangs.
*/
ksyn_handle_cvbroad(ckwq, uptoseq, &updatebits);
ckwq->kw_sword += (updatebits & PTHRW_COUNT_MASK);
} else if (nkwe == NULL) {
ksyn_wqunlock(ckwq);
nkwe = (ksyn_waitq_element_t) kzalloc(sizeof(struct ksyn_waitq_element), GFP_KERNEL); //zalloc(kwe_zone);
ksyn_wqlock(ckwq);
goto retry;
} else {
/* no eligible entries - add prepost */
bzero(nkwe, sizeof(struct ksyn_waitq_element));
sema_init(&nkwe->linux_sem, 0);
nkwe->kwe_kwqqueue = ckwq;
nkwe->kwe_flags = KWE_THREAD_PREPOST;
nkwe->kwe_lockseq = uptoseq;
nkwe->kwe_count = 1;
nkwe->kwe_uth = NULL;
nkwe->kwe_psynchretval = 0;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSIGNAL | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xfeedfefe, uptoseq, 0, 0);
#endif /* _PSYNCH_TRACE_ */
(void)ksyn_queue_insert(ckwq, &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER], uptoseq, NULL, nkwe, SEQFIT);
ckwq->kw_fakecount++;
nkwe = NULL;
}
/* set C or P bits and free if needed */
ksyn_cvupdate_fixup(ckwq, &updatebits, &kfreeq, 1);
}
ksyn_wqunlock(ckwq);
if (nkwe != NULL)
zfree(kwe_zone, nkwe);
ksyn_wqrelease(ckwq, NULL, 1, (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_CVAR));
out:
if (th != NULL)
thread_deallocate(th);
if (error == 0)
*retval = updatebits;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSIGNAL | DBG_FUNC_END, (uint32_t)cond, 0xeeeeeeed, updatebits, error, 0);
#endif /* _PSYNCH_TRACE_ */
return(error);
}
/*
* psynch_cvwait: This system call is used for psynch cvar waiters to block in kernel.
*/
int
darling_psynch_cvwait(__unused proc_t p, user_addr_t cond, uint64_t cvlsgen, uint32_t cugen, user_addr_t mutex, uint64_t mugen, uint32_t flags, int64_t sec, uint32_t nsec, uint32_t* retval)
{
uint32_t cgen, csgen;
uint32_t mgen, ugen;
ksyn_wait_queue_t kwq, ckwq;
int error=0, local_error = 0;
uint64_t abstime = 0;
uint32_t lockseq, updatebits=0;
struct linux_timespec ts;
uthread_t uth;
ksyn_waitq_element_t kwe, nkwe = NULL;
struct ksyn_queue *kq, kfreeq;
kern_return_t kret;
/* for conformance reasons */
__pthread_testcancel(0);
csgen = (uint32_t)((cvlsgen >> 32) & 0xffffffff);
cgen = ((uint32_t)(cvlsgen & 0xffffffff));
ugen = (uint32_t)((mugen >> 32) & 0xffffffff);
mgen = ((uint32_t)(mugen & 0xffffffff));
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVWAIT | DBG_FUNC_START, (uint32_t)cond, cgen, cugen, csgen, 0);
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVWAIT | DBG_FUNC_NONE, (uint32_t)mutex, mgen, ugen, flags, 0);
#endif /* _PSYNCH_TRACE_ */
lockseq = (cgen & PTHRW_COUNT_MASK);
/*
* In cvwait U word can be out of range as cond could be used only for
* timeouts. However S word needs to be within bounds and validated at
* user level as well.
*/
if (is_seqhigher_eq((csgen & PTHRW_COUNT_MASK), lockseq) != 0) {
__FAILEDUSERTEST__("psync_cvwait; invalid sequence numbers\n");
return LINUX_EINVAL;
}
ckwq = kwq = NULL;
error = ksyn_wqfind(cond, cgen, cugen, csgen, 0, flags, KSYN_WQTYPE_CVAR | KSYN_WQTYPE_INWAIT, &ckwq);
if (error != 0) {
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVWAIT | DBG_FUNC_END, (uint32_t)cond, 1, 0xdeadbeef, error, 0);
#endif /* _PSYNCH_TRACE_ */
return(error);
}
if (mutex != (user_addr_t)0) {
error = ksyn_wqfind(mutex, mgen, ugen, 0, 0, flags, (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_MTX), &kwq);
if (error != 0) {
local_error = error;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVWAIT | DBG_FUNC_END, (uint32_t)mutex, 2, 0xdeadbeef, error, 0);
#endif /* _PSYNCH_TRACE_ */
goto out;
}
(void)psynch_mutexdrop_internal(kwq, mgen, ugen, flags);
/* drops kwq reference */
kwq = NULL;
}
if (sec != 0 || (nsec & 0x3fffffff) != 0) {
ts.tv_sec = sec;
ts.tv_nsec = (nsec & 0x3fffffff);
nanoseconds_to_absolutetime((uint64_t)ts.tv_sec * NSEC_PER_SEC + ts.tv_nsec, &abstime );
clock_absolutetime_interval_to_deadline( abstime, &abstime );
}
ksyn_wqlock(ckwq);
/* update L, U and S... */
UPDATE_CVKWQ(ckwq, cgen, cugen, csgen, 0, KSYN_WQTYPE_CVAR);
/* Look for the sequence for prepost (or conflicting thread */
kq = &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER];
kwe = ksyn_queue_find_cvpreposeq(kq, lockseq);
if (kwe != NULL) {
switch (kwe->kwe_flags) {
case KWE_THREAD_INWAIT:
ksyn_wqunlock(ckwq);
__FAILEDUSERTEST__("cvwait: thread entry with same sequence already present\n");
local_error = LINUX_EBUSY;
goto out;
case KWE_THREAD_BROADCAST:
break;
case KWE_THREAD_PREPOST:
if ((kwe->kwe_lockseq & PTHRW_COUNT_MASK) == lockseq) {
/* we can safely consume a reference, so do so */
if (--kwe->kwe_count == 0) {
ksyn_queue_removeitem(ckwq, kq, kwe);
ckwq->kw_fakecount--;
nkwe = kwe;
}
} else {
/*
* consuming a prepost higher than our lock sequence is valid, but
* can leave the higher thread without a match. Convert the entry
* to a broadcast to compensate for this.
*/
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVWAIT | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xc2c2c2c2, kwe->kwe_lockseq, 0, 0);
#endif /* _PSYNCH_TRACE_ */
ksyn_handle_cvbroad(ckwq, kwe->kwe_lockseq, &updatebits);
#if __TESTPANICS__
if (updatebits != 0)
panic("psync_cvwait: convert pre-post to broadcast: woke up %d threads that shouldn't be there\n",
updatebits);
#endif /* __TESTPANICS__ */
}
break;
default:
panic("psync_cvwait: unexpected wait queue element type\n");
}
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVWAIT | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xfefefefe, kwe->kwe_lockseq, 0, 0);
#endif /* _PSYNCH_TRACE_ */
updatebits = PTHRW_INC;
ckwq->kw_sword += PTHRW_INC;
/* set C or P bits and free if needed */
ksyn_cvupdate_fixup(ckwq, &updatebits, &kfreeq, 1);
error = 0;
local_error = 0;
*retval = updatebits;
ksyn_wqunlock(ckwq);
if (nkwe != NULL)
zfree(kwe_zone, nkwe);
goto out;
}
uth = current_uthread();
kwe = &uth->uu_save.uus_kwe;
sema_init(&kwe->linux_sem, 0);
kwe->kwe_kwqqueue = ckwq;
kwe->kwe_flags = KWE_THREAD_INWAIT;
kwe->kwe_lockseq = lockseq;
kwe->kwe_count = 1;
kwe->kwe_uth = uth;
kwe->kwe_psynchretval = 0;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVWAIT | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xfeedfeed, cgen, 0, 0);
#endif /* _PSYNCH_TRACE_ */
error = ksyn_queue_insert(ckwq, kq, cgen, uth, kwe, SEQFIT);
if (error != 0) {
ksyn_wqunlock(ckwq);
local_error = error;
goto out;
}
kret = ksyn_block_thread_locked(ckwq, abstime, kwe, 1, psynch_cvcontinue, (void *)ckwq);
/* lock dropped */
psynch_cvcontinue(ckwq, kret);
/* not expected to return from unix_syscall_return */
panic("psynch_cvcontinue returned from unix_syscall_return");
out:
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVWAIT | DBG_FUNC_END, (uint32_t)cond, 0xeeeeeeed, (uint32_t)*retval, local_error, 0);
#endif /* _PSYNCH_TRACE_ */
ksyn_wqrelease(ckwq, NULL, 1, (KSYN_WQTYPE_INWAIT | KSYN_WQTYPE_CVAR));
return(local_error);
}
void
psynch_cvcontinue(void * parameter, kern_return_t result)
{
int error = 0, local_error = 0;
uthread_t uth = current_uthread();
ksyn_wait_queue_t ckwq = (ksyn_wait_queue_t)parameter;
ksyn_waitq_element_t kwe;
struct ksyn_queue kfreeq;
switch (result) {
case THREAD_TIMED_OUT:
error = LINUX_ETIMEDOUT;
break;
case THREAD_INTERRUPTED:
error = LINUX_EINTR;
break;
default:
error = 0;
break;
}
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_THWAKEUP | DBG_FUNC_NONE, 0xf4f3f2f1, (uintptr_t)uth, result, 0, 0);
#endif /* _PSYNCH_TRACE_ */
local_error = error;
kwe = &uth->uu_save.uus_kwe;
if (error != 0) {
ksyn_wqlock(ckwq);
/* just in case it got woken up as we were granting */
uth->uu_rval[0] = kwe->kwe_psynchretval;
#if __TESTPANICS__
if ((kwe->kwe_kwqqueue != NULL) && (kwe->kwe_kwqqueue != ckwq))
panic("cvwait waiting on some other kwq\n");
#endif /* __TESTPANICS__ */
if (kwe->kwe_kwqqueue != NULL) {
ksyn_queue_removeitem(ckwq, &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER], kwe);
kwe->kwe_kwqqueue = NULL;
}
if ((kwe->kwe_psynchretval & PTH_RWL_MTX_WAIT) != 0) {
/* the condition var granted.
* reset the error so that the thread returns back.
*/
local_error = 0;
/* no need to set any bits just return as cvsig/broad covers this */
ksyn_wqunlock(ckwq);
goto out;
}
ckwq->kw_sword += PTHRW_INC;
/* set C and P bits, in the local error */
if ((ckwq->kw_lword & PTHRW_COUNT_MASK) == (ckwq->kw_sword & PTHRW_COUNT_MASK)) {
local_error |= ECVCERORR;
if (ckwq->kw_inqueue != 0) {
(void)ksyn_queue_move_tofree(ckwq, &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER], (ckwq->kw_lword & PTHRW_COUNT_MASK), &kfreeq, 1, 1);
}
ckwq->kw_lword = ckwq->kw_uword = ckwq->kw_sword = 0;
ckwq->kw_kflags |= KSYN_KWF_ZEROEDOUT;
} else {
/* everythig in the queue is a fake entry ? */
if ((ckwq->kw_inqueue != 0) && (ckwq->kw_fakecount == ckwq->kw_inqueue)) {
local_error |= ECVPERORR;
}
}
ksyn_wqunlock(ckwq);
} else {
/* PTH_RWL_MTX_WAIT is removed */
if ((kwe->kwe_psynchretval & PTH_RWS_CV_MBIT) != 0)
uth->uu_rval[0] = PTHRW_INC | PTH_RWS_CV_CBIT;
else
uth->uu_rval[0] = 0;
local_error = 0;
}
out:
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVWAIT | DBG_FUNC_END, (uint32_t)ckwq->kw_addr, 0xeeeeeeed, uth->uu_rval[0], local_error, 0);
#endif /* _PSYNCH_TRACE_ */
ksyn_wqrelease(ckwq, NULL, 1, (KSYN_WQTYPE_INWAIT | KSYN_WQTYPE_CVAR));
unix_syscall_return(local_error);
}
/*
* psynch_cvclrprepost: This system call clears pending prepost if present.
*/
int
darling_psynch_cvclrprepost(__unused proc_t p, user_addr_t cond, uint32_t cgen, uint32_t cugen, uint32_t csgen, uint32_t pseq, uint32_t preposeq, uint32_t flags, int* retval)
{
int error;
ksyn_wait_queue_t ckwq = NULL;
struct ksyn_queue kfreeq;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CLRPRE | DBG_FUNC_START, (uint32_t)cond, cgen, cugen, csgen, 0);
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CLRPRE | DBG_FUNC_NONE, (uint32_t)cond, 0xcececece, pseq, flags, 0);
#endif /* _PSYNCH_TRACE_ */
if ((flags & _PTHREAD_MTX_OPT_MUTEX) == 0) {
error = ksyn_wqfind(cond, cgen, cugen, csgen, 0, flags, (KSYN_WQTYPE_CVAR | KSYN_WQTYPE_INDROP), &ckwq);
if (error != 0) {
*retval = 0;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CLRPRE | DBG_FUNC_END, (uint32_t)cond, 0, 0xdeadbeef, error, 0);
#endif /* _PSYNCH_TRACE_ */
return(error);
}
ksyn_wqlock(ckwq);
(void)ksyn_queue_move_tofree(ckwq, &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER], (pseq & PTHRW_COUNT_MASK), &kfreeq, 0, 1);
ksyn_wqunlock(ckwq);
ksyn_wqrelease(ckwq, NULL, 1, (KSYN_WQTYPE_CVAR | KSYN_WQTYPE_INDROP));
} else {
/* mutex type */
error = ksyn_wqfind(cond, cgen, cugen, 0, 0, flags, (KSYN_WQTYPE_MTX | KSYN_WQTYPE_INDROP), &ckwq);
if (error != 0) {
*retval = 0;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CLRPRE | DBG_FUNC_END, (uint32_t)cond, 0, 0xdeadbeef, error, 0);
#endif /* _PSYNCH_TRACE_ */
return(error);
}
ksyn_wqlock(ckwq);
if (((flags & _PTHREAD_MUTEX_POLICY_FIRSTFIT) != 0) && (ckwq->kw_pre_rwwc != 0)) {
if (is_seqlower_eq(ckwq->kw_pre_lockseq, cgen) != 0) {
/* clear prepost */
ckwq->kw_pre_rwwc = 0;
ckwq->kw_pre_lockseq = 0;
}
}
ksyn_wqunlock(ckwq);
ksyn_wqrelease(ckwq, NULL, 1, (KSYN_WQTYPE_MTX | KSYN_WQTYPE_INDROP));
}
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CLRPRE | DBG_FUNC_END, (uint32_t)cond, 0xeeeeeeed, 0, 0, 0);
#endif /* _PSYNCH_TRACE_ */
return(0);
}
/* ***************** pthread_rwlock ************************ */
/*
* psynch_rw_rdlock: This system call is used for psync rwlock readers to block.
*/
int
darling_psynch_rw_rdlock(__unused proc_t p, user_addr_t rwlock, uint32_t lgen, uint32_t ugen, uint32_t rw_wc, int flags, uint32_t* retval)
{
int error = 0, block;
uint32_t lockseq = 0, updatebits = 0, preseq = 0, prerw_wc = 0;
ksyn_wait_queue_t kwq;
uthread_t uth;
int isinit = lgen & PTHRW_RWL_INIT;
uint32_t returnbits = 0;
ksyn_waitq_element_t kwe;
kern_return_t kret;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWRDLOCK | DBG_FUNC_START, (uint32_t)rwlock, lgen, ugen, rw_wc, 0);
#endif /* _PSYNCH_TRACE_ */
uth = current_uthread();
/* preserve the seq number */
kwe = &uth->uu_save.uus_kwe;
sema_init(&kwe->linux_sem, 0);
kwe->kwe_lockseq = lgen;
kwe->kwe_uth = uth;
kwe->kwe_psynchretval = 0;
kwe->kwe_kwqqueue = NULL;
lockseq = lgen & PTHRW_COUNT_MASK;
error = ksyn_wqfind(rwlock, lgen, ugen, rw_wc, TID_ZERO, flags, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_RWLOCK), &kwq);
if (error != 0) {
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWRDLOCK | DBG_FUNC_END, (uint32_t)rwlock, 1, 0, error, 0);
#endif /* _PSYNCH_TRACE_ */
return(error);
}
ksyn_wqlock(kwq);
if (isinit != 0) {
lgen &= ~PTHRW_RWL_INIT;
if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) == 0) {
/* first to notice the reset of the lock, clear preposts */
CLEAR_REINIT_BITS(kwq);
kwq->kw_kflags |= KSYN_KWF_INITCLEARED;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 1, 0);
#endif /* _PSYNCH_TRACE_ */
}
}
/* handle first the missed wakeups */
if ((kwq->kw_pre_intrcount != 0) &&
((kwq->kw_pre_intrtype == PTH_RW_TYPE_READ) || (kwq->kw_pre_intrtype == PTH_RW_TYPE_LREAD)) &&
(is_seqlower_eq(lockseq, (kwq->kw_pre_intrseq & PTHRW_COUNT_MASK)) != 0)) {
kwq->kw_pre_intrcount--;
kwe->kwe_psynchretval = kwq->kw_pre_intrretbits;
if (kwq->kw_pre_intrcount==0)
CLEAR_INTR_PREPOST_BITS(kwq);
ksyn_wqunlock(kwq);
goto out;
}
/* handle overlap first as they are not counted against pre_rwwc */
/* check for overlap and if no pending W bit (indicates writers) */
if ((kwq->kw_overlapwatch != 0) && ((rw_wc & PTHRW_RWS_SAVEMASK) == 0) && ((lgen & PTH_RWL_WBIT) == 0)) {
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWRDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 10, kwq->kw_nextseqword, kwq->kw_lastseqword, 0);
#endif /* _PSYNCH_TRACE_ */
error = kwq_handle_overlap(kwq, lgen, ugen, rw_wc, &updatebits, (KW_UNLOCK_PREPOST_READLOCK|KW_UNLOCK_PREPOST), &block);
#if __TESTPANICS__
if (error != 0)
panic("rw_rdlock: kwq_handle_overlap failed %d\n",error);
#endif /* __TESTPANICS__ */
if (block == 0) {
error = 0;
kwe->kwe_psynchretval = updatebits;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWRDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 0xff, updatebits, 0xee, 0);
#endif /* _PSYNCH_TRACE_ */
ksyn_wqunlock(kwq);
goto out;
}
}
if ((kwq->kw_pre_rwwc != 0) && (is_seqlower_eq(lockseq, (kwq->kw_pre_lockseq & PTHRW_COUNT_MASK)) != 0)) {
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWRDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 2, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
#endif /* _PSYNCH_TRACE_ */
kwq->kw_pre_rwwc--;
if (kwq->kw_pre_rwwc == 0) {
preseq = kwq->kw_pre_lockseq;
prerw_wc = kwq->kw_pre_sseq;
CLEAR_PREPOST_BITS(kwq);
if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) != 0){
kwq->kw_kflags &= ~KSYN_KWF_INITCLEARED;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 0, 0);
#endif /* _PSYNCH_TRACE_ */
}
error = kwq_handle_unlock(kwq, preseq, prerw_wc, &updatebits, (KW_UNLOCK_PREPOST_READLOCK|KW_UNLOCK_PREPOST), &block, lgen);
#if __TESTPANICS__
if (error != 0)
panic("rw_rdlock: kwq_handle_unlock failed %d\n",error);
#endif /* __TESTPANICS__ */
if (block == 0) {
ksyn_wqunlock(kwq);
goto out;
}
/* insert to q and proceed as ususal */
}
}
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWRDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 3, 0, 0, 0);
#endif /* _PSYNCH_TRACE_ */
error = ksyn_queue_insert(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_READ], lgen, uth, kwe, SEQFIT);
#if __TESTPANICS__
if (error != 0)
panic("psynch_rw_rdlock: failed to enqueue\n");
#endif /* __TESTPANICS__ */
kret = ksyn_block_thread_locked(kwq, (uint64_t)0, kwe, 0, THREAD_CONTINUE_NULL, NULL);
/* drops the kwq lock */
switch (kret) {
case THREAD_TIMED_OUT:
error = LINUX_ETIMEDOUT;
break;
case THREAD_INTERRUPTED:
error = LINUX_EINTR;
break;
default:
error = 0;
break;
}
out:
if (error != 0) {
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWRDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 4, error, 0, 0);
#endif /* _PSYNCH_TRACE_ */
ksyn_wqlock(kwq);
if (kwe->kwe_kwqqueue != NULL)
ksyn_queue_removeitem(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_READ], kwe);
ksyn_wqunlock(kwq);
} else {
/* update bits */
*retval = kwe->kwe_psynchretval;
returnbits = kwe->kwe_psynchretval;
}
ksyn_wqrelease(kwq, NULL, 0, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_RWLOCK));
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWRDLOCK | DBG_FUNC_END, (uint32_t)rwlock, 1, returnbits, error, 0);
#endif /* _PSYNCH_TRACE_ */
return(error);
}
/*
* psynch_rw_longrdlock: This system call is used for psync rwlock long readers to block.
*/
int
#ifdef NOTYET
darling_psynch_rw_longrdlock(__unused proc_t p, user_addr_t rwlock, uint32_t lgen, uint32_t ugen, uint32_t rw_wc, int flags, uint32_t* retval)
#else /* NOTYET */
darling_psynch_rw_longrdlock(__unused proc_t p, __unused user_addr_t rwlock, __unused uint32_t lgenval, __unused uint32_t ugenval, __unused uint32_t rw_wc, __unused int flags, __unused uint32_t* retval)
#endif /* NOTYET */
{
#ifdef NOTYET
int isinit = lgen & PTHRW_RWL_INIT;
uint32_t returnbits=0;
ksyn_waitq_element_t kwe;
kern_return_t kret;
ksyn_wait_queue_t kwq;
int error=0, block = 0 ;
uthread_t uth;
uint32_t lockseq = 0, updatebits = 0, preseq = 0, prerw_wc = 0;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWLRDLOCK | DBG_FUNC_START, (uint32_t)rwlock, lgen, ugen, rw_wc, 0);
#endif /* _PSYNCH_TRACE_ */
uth = current_uthread();
kwe = &uth->uu_save.uus_kwe;
kwe->kwe_lockseq = lgen;
kwe->kwe_uth = uth;
kwe->kwe_psynchretval = 0;
kwe->kwe_kwqqueue = NULL;
lockseq = (lgen & PTHRW_COUNT_MASK);
error = ksyn_wqfind(rwlock, lgen, ugen, rw_wc, TID_ZERO, flags, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_RWLOCK), &kwq);
if (error != 0) {
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWLRDLOCK | DBG_FUNC_END, (uint32_t)rwlock, 1, 0, error, 0);
#endif /* _PSYNCH_TRACE_ */
return(error);
}
ksyn_wqlock(kwq);
if (isinit != 0) {
lgen &= ~PTHRW_RWL_INIT;
if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) == 0) {
/* first to notice the reset of the lock, clear preposts */
CLEAR_REINIT_BITS(kwq);
kwq->kw_kflags |= KSYN_KWF_INITCLEARED;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 1, 0);
#endif /* _PSYNCH_TRACE_ */
}
}
/* handle first the missed wakeups */
if ((kwq->kw_pre_intrcount != 0) &&
(kwq->kw_pre_intrtype == PTH_RW_TYPE_LREAD) &&
(is_seqlower_eq(lockseq, (kwq->kw_pre_intrseq & PTHRW_COUNT_MASK)) != 0)) {
kwq->kw_pre_intrcount--;
kwe->kwe_psynchretval = kwq->kw_pre_intrretbits;
if (kwq->kw_pre_intrcount==0)
CLEAR_INTR_PREPOST_BITS(kwq);
ksyn_wqunlock(kwq);
goto out;
}
if ((kwq->kw_pre_rwwc != 0) && (is_seqlower_eq(lockseq, (kwq->kw_pre_lockseq & PTHRW_COUNT_MASK)) != 0)) {
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWLRDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 2, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
#endif /* _PSYNCH_TRACE_ */
kwq->kw_pre_rwwc--;
if (kwq->kw_pre_rwwc == 0) {
preseq = kwq->kw_pre_lockseq;
prerw_wc = kwq->kw_pre_sseq;
CLEAR_PREPOST_BITS(kwq);
if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) != 0){
kwq->kw_kflags &= ~KSYN_KWF_INITCLEARED;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 0, 0);
#endif /* _PSYNCH_TRACE_ */
}
error = kwq_handle_unlock(kwq, preseq, prerw_wc, &updatebits, (KW_UNLOCK_PREPOST_LREADLOCK|KW_UNLOCK_PREPOST), &block, lgen);
#if __TESTPANICS__
if (error != 0)
panic("kwq_handle_unlock failed %d\n",error);
#endif /* __TESTPANICS__ */
if (block == 0) {
ksyn_wqunlock(kwq);
goto out;
}
/* insert to q and proceed as ususal */
}
}
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWLRDLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 3, 0, 0, 0);
#endif /* _PSYNCH_TRACE_ */
error = ksyn_queue_insert(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_LREAD], lgen, uth, kwe, SEQFIT);
#if __TESTPANICS__
if (error != 0)
panic("psynch_rw_longrdlock: failed to enqueue\n");
#endif /* __TESTPANICS__ */
kret = ksyn_block_thread_locked(kwq, (uint64_t)0, kwe, 0, THREAD_CONTINUE_NULL, NULL);
/* drops the kwq lock */
switch (kret) {
case THREAD_TIMED_OUT:
error = ETIMEDOUT;
break;
case THREAD_INTERRUPTED:
error = EINTR;
break;
default:
error = 0;
break;
}
out:
if (error != 0) {
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWLRDLOCK | DBG_FUNC_END, (uint32_t)rwlock, 1, 0, error, 0);
#endif /* _PSYNCH_TRACE_ */
ksyn_wqlock(kwq);
if (kwe->kwe_kwqqueue != NULL)
ksyn_queue_removeitem(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_LREAD], kwe);
ksyn_wqunlock(kwq);
} else {
/* update bits */
*retval = kwe->kwe_psynchretval;
returnbits = kwe->kwe_psynchretval;
}
ksyn_wqrelease(kwq, NULL, 0, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_RWLOCK));
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWLRDLOCK | DBG_FUNC_END, (uint32_t)rwlock, 0, returnbits, error, 0);
#endif /* _PSYNCH_TRACE_ */
return(error);
#else /* NOTYET */
return(LINUX_ESRCH);
#endif /* NOTYET */
}
/*
* psynch_rw_wrlock: This system call is used for psync rwlock writers to block.
*/
int
darling_psynch_rw_wrlock(__unused proc_t p, user_addr_t rwlock, uint32_t lgen, uint32_t ugen, uint32_t rw_wc, int flags, uint32_t* retval)
{
int block;
ksyn_wait_queue_t kwq;
int error=0;
uthread_t uth;
uint32_t lockseq = 0, updatebits = 0, preseq = 0, prerw_wc = 0;
int isinit = lgen & PTHRW_RWL_INIT;
uint32_t returnbits = 0;
ksyn_waitq_element_t kwe;
kern_return_t kret;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWWRLOCK | DBG_FUNC_START, (uint32_t)rwlock, lgen, ugen, rw_wc, 0);
#endif /* _PSYNCH_TRACE_ */
uth = current_uthread();
kwe = &uth->uu_save.uus_kwe;
sema_init(&kwe->linux_sem, 0);
kwe->kwe_lockseq = lgen;
kwe->kwe_uth = uth;
kwe->kwe_psynchretval = 0;
kwe->kwe_kwqqueue = NULL;
lockseq = (lgen & PTHRW_COUNT_MASK);
error = ksyn_wqfind(rwlock, lgen, ugen, rw_wc, TID_ZERO, flags, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_RWLOCK), &kwq);
if (error != 0) {
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWWRLOCK | DBG_FUNC_END, (uint32_t)rwlock, 1, 0, error, 0);
#endif /* _PSYNCH_TRACE_ */
return(error);
}
ksyn_wqlock(kwq);
if (isinit != 0) {
lgen &= ~PTHRW_RWL_INIT;
if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) == 0) {
/* first to notice the reset of the lock, clear preposts */
CLEAR_REINIT_BITS(kwq);
kwq->kw_kflags |= KSYN_KWF_INITCLEARED;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 1, 0);
#endif /* _PSYNCH_TRACE_ */
}
}
/* handle first the missed wakeups */
if ((kwq->kw_pre_intrcount != 0) &&
(kwq->kw_pre_intrtype == PTH_RW_TYPE_WRITE) &&
(is_seqlower_eq(lockseq, (kwq->kw_pre_intrseq & PTHRW_COUNT_MASK)) != 0)) {
kwq->kw_pre_intrcount--;
kwe->kwe_psynchretval = kwq->kw_pre_intrretbits;
if (kwq->kw_pre_intrcount==0)
CLEAR_INTR_PREPOST_BITS(kwq);
ksyn_wqunlock(kwq);
goto out;
}
if ((kwq->kw_pre_rwwc != 0) && (is_seqlower_eq(lockseq, (kwq->kw_pre_lockseq & PTHRW_COUNT_MASK)) != 0)) {
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWWRLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 2, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
#endif /* _PSYNCH_TRACE_ */
kwq->kw_pre_rwwc--;
if (kwq->kw_pre_rwwc == 0) {
preseq = kwq->kw_pre_lockseq;
prerw_wc = kwq->kw_pre_sseq;
CLEAR_PREPOST_BITS(kwq);
if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) != 0){
kwq->kw_kflags &= ~KSYN_KWF_INITCLEARED;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 0, 0);
#endif /* _PSYNCH_TRACE_ */
}
error = kwq_handle_unlock(kwq, preseq, prerw_wc, &updatebits, (KW_UNLOCK_PREPOST_WRLOCK|KW_UNLOCK_PREPOST), &block, lgen);
#if __TESTPANICS__
if (error != 0)
panic("rw_wrlock: kwq_handle_unlock failed %d\n",error);
#endif /* __TESTPANICS__ */
if (block == 0) {
ksyn_wqunlock(kwq);
*retval = updatebits;
goto out1;
}
/* insert to q and proceed as ususal */
}
}
/* No overlap watch needed go ahead and block */
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWWRLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 3, 0, 0, 0);
#endif /* _PSYNCH_TRACE_ */
error = ksyn_queue_insert(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], lgen, uth, kwe, SEQFIT);
#if __TESTPANICS__
if (error != 0)
panic("psynch_rw_wrlock: failed to enqueue\n");
#endif /* __TESTPANICS__ */
kret = ksyn_block_thread_locked(kwq, (uint64_t)0, kwe, 0, THREAD_CONTINUE_NULL, NULL);
/* drops the wq lock */
switch (kret) {
case THREAD_TIMED_OUT:
error = LINUX_ETIMEDOUT;
break;
case THREAD_INTERRUPTED:
error = LINUX_EINTR;
break;
default:
error = 0;
break;
}
out:
if (error != 0) {
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWWRLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 4, error, 0, 0);
#endif /* _PSYNCH_TRACE_ */
ksyn_wqlock(kwq);
if (kwe->kwe_kwqqueue != NULL)
ksyn_queue_removeitem(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], kwe);
ksyn_wqunlock(kwq);
} else {
/* update bits */
*retval = kwe->kwe_psynchretval;
returnbits = kwe->kwe_psynchretval;
}
out1:
ksyn_wqrelease(kwq, NULL, 0, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_RWLOCK));
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWWRLOCK | DBG_FUNC_END, (uint32_t)rwlock, 1, returnbits, error, 0);
#endif /* _PSYNCH_TRACE_ */
return(error);
}
/*
* psynch_rw_yieldwrlock: This system call is used for psync rwlock yielding writers to block.
*/
int
#ifdef NOTYET
darling_psynch_rw_yieldwrlock(__unused proc_t p, user_addr_t rwlock, uint32_t lgen, uint32_t ugen, uint32_t rw_wc, int flags, uint32_t* retval)
#else /* NOTYET */
darling_psynch_rw_yieldwrlock(__unused proc_t p, __unused user_addr_t rwlock, __unused uint32_t lgen, __unused uint32_t ugen, __unused uint32_t rw_wc, __unused int flags, __unused uint32_t* retval)
#endif /* NOTYET */
{
#ifdef NOTYET
int block;
ksyn_wait_queue_t kwq;
int error=0;
int isinit = lgen & PTHRW_RWL_INIT;
uthread_t uth;
uint32_t returnbits=0;
ksyn_waitq_element_t kwe;
kern_return_t kret;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWYWRLOCK | DBG_FUNC_START, (uint32_t)rwlock, lgen, ugen, rw_wc, 0);
#endif /* _PSYNCH_TRACE_ */
uint32_t lockseq = 0, updatebits = 0, preseq = 0, prerw_wc = 0;
uth = current_uthread();
kwe = &uth->uu_save.uus_kwe;
kwe->kwe_lockseq = lgen;
kwe->kwe_uth = uth;
kwe->kwe_psynchretval = 0;
kwe->kwe_kwqqueue = NULL;
lockseq = (lgen & PTHRW_COUNT_MASK);
error = ksyn_wqfind(rwlock, lgen, ugen, rw_wc, TID_ZERO, flags, (KSYN_WQTYPE_INWAIT|KSYN_WQTYPE_RWLOCK), &kwq);
if (error != 0) {
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWYWRLOCK | DBG_FUNC_END, (uint32_t)rwlock, 1, 0, error, 0);
#endif /* _PSYNCH_TRACE_ */
return(error);
}
ksyn_wqlock(kwq);
if (isinit != 0) {
lgen &= ~PTHRW_RWL_INIT;
if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) == 0) {
/* first to notice the reset of the lock, clear preposts */
CLEAR_REINIT_BITS(kwq);
kwq->kw_kflags |= KSYN_KWF_INITCLEARED;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 1, 0);
#endif /* _PSYNCH_TRACE_ */
}
}
/* handle first the missed wakeups */
if ((kwq->kw_pre_intrcount != 0) &&
(kwq->kw_pre_intrtype == PTH_RW_TYPE_YWRITE) &&
(is_seqlower_eq(lockseq, (kwq->kw_pre_intrseq & PTHRW_COUNT_MASK)) != 0)) {
kwq->kw_pre_intrcount--;
kwe->kwe_psynchretval = kwq->kw_pre_intrretbits;
if (kwq->kw_pre_intrcount==0)
CLEAR_INTR_PREPOST_BITS(kwq);
ksyn_wqunlock(kwq);
goto out;
}
if ((kwq->kw_pre_rwwc != 0) && (is_seqlower_eq(lockseq, (kwq->kw_pre_lockseq & PTHRW_COUNT_MASK)) != 0)) {
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWYWRLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 2, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
#endif /* _PSYNCH_TRACE_ */
kwq->kw_pre_rwwc--;
if (kwq->kw_pre_rwwc == 0) {
preseq = kwq->kw_pre_lockseq;
prerw_wc = kwq->kw_pre_sseq;
CLEAR_PREPOST_BITS(kwq);
if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) != 0){
kwq->kw_kflags &= ~KSYN_KWF_INITCLEARED;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 0, 0);
#endif /* _PSYNCH_TRACE_ */
}
error = kwq_handle_unlock(kwq, preseq, prerw_wc, &updatebits, (KW_UNLOCK_PREPOST_YWRLOCK|KW_UNLOCK_PREPOST), &block, lgen);
#if __TESTPANICS__
if (error != 0)
panic("kwq_handle_unlock failed %d\n",error);
#endif /* __TESTPANICS__ */
if (block == 0) {
ksyn_wqunlock(kwq);
*retval = updatebits;
goto out;
}
/* insert to q and proceed as ususal */
}
}
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWYWRLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 3, 0, 0, 0);
#endif /* _PSYNCH_TRACE_ */
error = ksyn_queue_insert(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER], lgen, uth, kwe, SEQFIT);
#if __TESTPANICS__
if (error != 0)
panic("psynch_rw_yieldwrlock: failed to enqueue\n");
#endif /* __TESTPANICS__ */
kret = ksyn_block_thread_locked(kwq, (uint64_t)0, kwe, 0, THREAD_CONTINUE_NULL, NULL);
switch (kret) {
case THREAD_TIMED_OUT:
error = ETIMEDOUT;
break;
case THREAD_INTERRUPTED:
error = EINTR;
break;
default:
error = 0;
break;
}
out:
if (error != 0) {
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWYWRLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 4, error, 0, 0);
#endif /* _PSYNCH_TRACE_ */
ksyn_wqlock(kwq);
if (kwe->kwe_kwqqueue != NULL)
ksyn_queue_removeitem(kwq, &kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER], kwe);
ksyn_wqunlock(kwq);
} else {
/* update bits */
*retval = kwe->kwe_psynchretval;
returnbits = kwe->kwe_psynchretval;
}
ksyn_wqrelease(kwq, NULL, 0, (KSYN_WQTYPE_INWAIT | KSYN_WQTYPE_RWLOCK));
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWYWRLOCK | DBG_FUNC_END, (uint32_t)rwlock, 1, returnbits, error, 0);
#endif /* _PSYNCH_TRACE_ */
return(error);
#else /* NOTYET */
return(LINUX_ESRCH);
#endif /* NOTYET */
}
/*
* psynch_rw_unlock: This system call is used for unlock state postings. This will grant appropriate
* reader/writer variety lock.
*/
int
darling_psynch_rw_unlock(__unused proc_t p, user_addr_t rwlock, uint32_t lgen, uint32_t ugen, uint32_t rw_wc, int flags, uint32_t* retval)
{
uint32_t curgen;
uthread_t uth;
ksyn_wait_queue_t kwq;
uint32_t updatebits = 0;
int error=0, diff;
uint32_t count = 0;
int isinit = 0;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUNLOCK | DBG_FUNC_START, (uint32_t)rwlock, lgen, ugen, rw_wc, 0);
#endif /* _PSYNCH_TRACE_ */
uth = current_uthread();
error = ksyn_wqfind(rwlock, lgen, ugen, rw_wc, TID_ZERO, flags, (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_RWLOCK), &kwq);
if (error != 0) {
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUNLOCK | DBG_FUNC_END, (uint32_t)rwlock, 1, 0, error, 0);
#endif /* _PSYNCH_TRACE_ */
return(error);
}
curgen = lgen & PTHRW_COUNT_MASK;
ksyn_wqlock(kwq);
if ((lgen & PTHRW_RWL_INIT) != 0) {
lgen &= ~PTHRW_RWL_INIT;
if ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) == 0){
CLEAR_REINIT_BITS(kwq);
kwq->kw_kflags |= KSYN_KWF_INITCLEARED;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 1, 0);
#endif /* _PSYNCH_TRACE_ */
}
isinit = 1;
}
/* if lastunlock seq is set, ensure the current one is not lower than that, as it would be spurious */
if ((kwq->kw_lastunlockseq != PTHRW_RWL_INIT) && (is_seqlower(ugen, kwq->kw_lastunlockseq)!= 0)) {
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, (uint32_t)0xeeeeeeee, rw_wc, kwq->kw_lastunlockseq, 0);
#endif /* _PSYNCH_TRACE_ */
error = 0;
goto out;
}
/* If L-U != num of waiters, then it needs to be preposted or spr */
diff = find_diff(lgen, ugen);
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 1, kwq->kw_inqueue, curgen, 0);
#endif /* _PSYNCH_TRACE_ */
if (find_seq_till(kwq, curgen, diff, &count) == 0) {
if ((count == 0) || (count < (uint32_t)diff))
goto prepost;
}
/* no prepost and all threads are in place, reset the bit */
if ((isinit != 0) && ((kwq->kw_kflags & KSYN_KWF_INITCLEARED) != 0)){
kwq->kw_kflags &= ~KSYN_KWF_INITCLEARED;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, lgen, ugen, rw_wc, 0, 0);
#endif /* _PSYNCH_TRACE_ */
}
/* can handle unlock now */
CLEAR_PREPOST_BITS(kwq);
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 2, 0, 0, 0);
#endif /* _PSYNCH_TRACE_ */
error = kwq_handle_unlock(kwq, lgen, rw_wc, &updatebits, 0, NULL, 0);
#if __TESTPANICS__
if (error != 0)
panic("psynch_rw_unlock: kwq_handle_unlock failed %d\n",error);
#endif /* __TESTPANICS__ */
out:
if (error == 0) {
/* update bits?? */
*retval = updatebits;
}
ksyn_wqunlock(kwq);
ksyn_wqrelease(kwq, NULL, 0, (KSYN_WQTYPE_INDROP | KSYN_WQTYPE_RWLOCK));
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUNLOCK | DBG_FUNC_END, (uint32_t)rwlock, 0, updatebits, error, 0);
#endif /* _PSYNCH_TRACE_ */
return(error);
prepost:
/* update if the new seq is higher than prev prepost, or first set */
if ((is_rws_setseq(kwq->kw_pre_sseq) != 0) ||
(is_seqhigher_eq((rw_wc & PTHRW_COUNT_MASK), (kwq->kw_pre_sseq & PTHRW_COUNT_MASK)) != 0)) {
kwq->kw_pre_rwwc = (diff - count);
kwq->kw_pre_lockseq = curgen;
kwq->kw_pre_sseq = rw_wc;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 3, rw_wc, count, 0);
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWUNLOCK | DBG_FUNC_NONE, (uint32_t)rwlock, 4, kwq->kw_pre_rwwc, kwq->kw_pre_lockseq, 0);
#endif /* _PSYNCH_TRACE_ */
updatebits = lgen; /* let this not do unlock handling */
}
error = 0;
goto out;
}
extern void* hashinit(int elements, int type, u_long* hashmask);
static void* darling_psynch_hashinit(int elements, int type, u_long* hashmask) {
struct pthhashhead* hashtbl = hashinit(elements, type, hashmask);
if (hashtbl) {
long hashsize = *hashmask + 1;
for (int i = 0; i < hashsize; i++)
LIST_INIT(&hashtbl[i]);
}
return hashtbl;
};
void
psynch_init(void)
{
pth_global_hashinit();
psynch_thcall = thread_call_allocate(psynch_wq_cleanup, NULL);
}
void
psynch_exit(void)
{
thread_call_free(psynch_thcall);
pth_global_hashexit();
}
/* ************************************************************************** */
void
pth_global_hashinit()
{
int arg;
pth_glob_hashtbl = darling_psynch_hashinit(PTH_HASHSIZE * 4, M_PROC, &pthhash);
#ifndef __DARLING__
/*
* pthtest={0,1,2,3} (override default aborting behavior on pthread sync failures)
* 0 - just return errors
* 1 - print and return errors
* 2 - abort user, print and return errors
* 3 - panic
*/
if (!PE_parse_boot_argn("pthtest", &arg, sizeof(arg)))
arg = __TESTMODE__;
if (arg == 3) {
__test_panics__ = 1;
printf("Pthread support PANICS when sync kernel primitives misused\n");
} else if (arg == 2) {
__test_aborts__ = 1;
__test_prints__ = 1;
printf("Pthread support ABORTS when sync kernel primitives misused\n");
} else if (arg == 1) {
__test_prints__ = 1;
printf("Pthread support LOGS when sync kernel primitives misused\n");
}
#endif
}
void
pth_global_hashexit()
{
FREE(pth_glob_hashtbl, M_PROC);
pth_glob_hashtbl = NULL;
}
void
darling_pth_proc_hashinit(proc_t p)
{
p->p_pthhash = darling_psynch_hashinit(PTH_HASHSIZE, M_PROC, &pthhash);
if (p->p_pthhash == NULL)
panic("pth_proc_hashinit: hash init returned 0\n");
}
ksyn_wait_queue_t
ksyn_wq_hash_lookup(user_addr_t mutex, proc_t p, int flags, uint64_t object, uint64_t objoffset)
{
ksyn_wait_queue_t kwq;
struct pthhashhead * hashptr;
if ((flags & PTHREAD_PSHARED_FLAGS_MASK) == PTHREAD_PROCESS_SHARED)
{
hashptr = pth_glob_hashtbl;
#ifndef __DARLING__
kwq = (&hashptr[object & pthhash])->lh_first;
if (kwq != 0) {
for (; kwq != NULL; kwq = kwq->kw_hash.le_next) {
if ((kwq->kw_object == object) &&(kwq->kw_offset == objoffset)) {
return (kwq);
}
}
}
#else
struct list_head* head = (&hashptr[object & pthhash]);
if (!list_empty(head)) {
list_for_each_entry(kwq, head, kw_hash) {
if ((kwq->kw_object == object) &&(kwq->kw_offset == objoffset)) {
return (kwq);
}
}
}
#endif
} else {
hashptr = p->p_pthhash;
#ifndef __DARLING__
kwq = (&hashptr[mutex & pthhash])->lh_first;
if (kwq != 0)
for (; kwq != NULL; kwq = kwq->kw_hash.le_next) {
if (kwq->kw_addr == mutex) {
return (kwq);
}
}
#else
struct list_head* head = (&hashptr[mutex & pthhash]);
if (!list_empty(head)) {
list_for_each_entry(kwq, head, kw_hash) {
if (kwq->kw_addr == mutex) {
return (kwq);
}
}
}
#endif
}
return(NULL);
}
void
darling_pth_proc_hashdelete(proc_t p)
{
struct pthhashhead * hashptr;
ksyn_wait_queue_t kwq;
int hashsize = pthhash + 1;
int i;
#if _PSYNCH_TRACE_
if ((pthread_debug_proc != NULL) && (p == pthread_debug_proc))
pthread_debug_proc = PROC_NULL;
#endif /* _PSYNCH_TRACE_ */
hashptr = p->p_pthhash;
p->p_pthhash = NULL;
if (hashptr == NULL)
return;
pthread_list_lock();
for(i= 0; i < hashsize; i++) {
while ((kwq = LIST_FIRST(&hashptr[i], kw_hash)) != NULL) {
if ((kwq->kw_pflags & KSYN_WQ_INHASH) != 0) {
kwq->kw_pflags &= ~KSYN_WQ_INHASH;
LIST_REMOVE(kwq, kw_hash);
}
if ((kwq->kw_pflags & KSYN_WQ_FLIST) != 0) {
kwq->kw_pflags &= ~KSYN_WQ_FLIST;
LIST_REMOVE(kwq, kw_list);
num_infreekwq--;
}
num_freekwq++;
pthread_list_unlock();
/* release fake entries if present for cvars */
if (((kwq->kw_type & KSYN_WQTYPE_MASK) == KSYN_WQTYPE_CVAR) && (kwq->kw_inqueue != 0))
ksyn_freeallkwe(&kwq->kw_ksynqueues[KSYN_QUEUE_WRITER]);
#ifndef __DARLING__
lck_mtx_destroy(&kwq->kw_lock, pthread_lck_grp);
#endif
zfree(kwq_zone, kwq);
pthread_list_lock();
}
}
pthread_list_unlock();
FREE(hashptr, M_PROC);
}
/* no lock held for this as the waitqueue is getting freed */
void
ksyn_freeallkwe(ksyn_queue_t kq)
{
ksyn_waitq_element_t kwe;
/* free all the fake entries, dequeue rest */
kwe = TAILQ_FIRST(&kq->ksynq_kwelist);
while (kwe != NULL) {
if (kwe->kwe_flags != KWE_THREAD_INWAIT) {
TAILQ_REMOVE(&kq->ksynq_kwelist, kwe, kwe_list);
zfree(kwe_zone, kwe);
} else {
TAILQ_REMOVE(&kq->ksynq_kwelist, kwe, kwe_list);
}
kwe = TAILQ_FIRST(&kq->ksynq_kwelist);
}
}
/* find kernel waitqueue, if not present create one. Grants a reference */
int
ksyn_wqfind(user_addr_t mutex, uint32_t mgen, uint32_t ugen, uint32_t rw_wc, uint64_t tid, int flags, int wqtype, ksyn_wait_queue_t * kwqp)
{
ksyn_wait_queue_t kwq;
ksyn_wait_queue_t nkwq;
struct pthhashhead * hashptr;
uint64_t object = 0, offset = 0;
uint64_t hashhint;
proc_t p = current_proc();
int retry = mgen & PTH_RWL_RETRYBIT;
struct ksyn_queue kfreeq;
int i;
if ((flags & PTHREAD_PSHARED_FLAGS_MASK) == PTHREAD_PROCESS_SHARED)
{
(void)ksyn_findobj(mutex, &object, &offset);
hashhint = object;
hashptr = pth_glob_hashtbl;
} else {
hashptr = p->p_pthhash;
}
ksyn_queue_init(&kfreeq);
if (((wqtype & KSYN_WQTYPE_MASK) == KSYN_WQTYPE_MTX) && (retry != 0))
mgen &= ~PTH_RWL_RETRYBIT;
loop:
//pthread_list_lock_spin();
pthread_list_lock();
kwq = ksyn_wq_hash_lookup(mutex, p, flags, object, offset);
if (kwq != NULL) {
if ((kwq->kw_pflags & KSYN_WQ_FLIST) != 0) {
LIST_REMOVE(kwq, kw_list);
kwq->kw_pflags &= ~KSYN_WQ_FLIST;
num_infreekwq--;
num_reusekwq++;
}
if ((kwq->kw_type & KSYN_WQTYPE_MASK) != (wqtype &KSYN_WQTYPE_MASK)) {
if ((kwq->kw_inqueue == 0) && (kwq->kw_pre_rwwc ==0) && (kwq->kw_pre_intrcount == 0)) {
if (kwq->kw_iocount == 0) {
kwq->kw_addr = mutex;
kwq->kw_flags = flags;
kwq->kw_object = object;
kwq->kw_offset = offset;
kwq->kw_type = (wqtype & KSYN_WQTYPE_MASK);
CLEAR_REINIT_BITS(kwq);
CLEAR_INTR_PREPOST_BITS(kwq);
CLEAR_PREPOST_BITS(kwq);
kwq->kw_lword = mgen;
kwq->kw_uword = ugen;
kwq->kw_sword = rw_wc;
kwq->kw_owner = tid;
} else if ((kwq->kw_iocount == 1) && (kwq->kw_dropcount == kwq->kw_iocount)) {
/* if all users are unlockers then wait for it to finish */
kwq->kw_pflags |= KSYN_WQ_WAITING;
/* wait for the wq to be free */
#ifndef __DARLING__
(void)msleep(&kwq->kw_pflags, pthread_list_mlock, PDROP, "ksyn_wqfind", 0);
#else
pthread_list_unlock();
if (wait_event_interruptible(kwq->linux_wq, !(kwq->kw_pflags & KSYN_WQ_WAITING) || darling_thread_canceled()) != 0
|| darling_thread_canceled())
{
return LINUX_EINTR;
}
#endif
/* does not have list lock */
goto loop;
} else {
__FAILEDUSERTEST__("address already known to kernel for another (busy) synchronizer type\n");
pthread_list_unlock();
return LINUX_EBUSY;
}
} else {
__FAILEDUSERTEST__("address already known to kernel for another (busy) synchronizer type(1)\n");
pthread_list_unlock();
return LINUX_EBUSY;
}
}
kwq->kw_iocount++;
if (wqtype == KSYN_WQTYPE_MUTEXDROP)
kwq->kw_dropcount++;
if (kwqp != NULL)
*kwqp = kwq;
pthread_list_unlock();
return (0);
}
pthread_list_unlock();
nkwq = (ksyn_wait_queue_t) kmalloc(sizeof(struct ksyn_wait_queue), GFP_KERNEL); //zalloc(kwq_zone);
bzero(nkwq, sizeof(struct ksyn_wait_queue));
mutex_init(&nkwq->kw_lock);
init_waitqueue_head(&nkwq->linux_wq);
nkwq->kw_addr = mutex;
nkwq->kw_flags = flags;
nkwq->kw_iocount = 1;
if (wqtype == KSYN_WQTYPE_MUTEXDROP)
nkwq->kw_dropcount++;
nkwq->kw_object = object;
nkwq->kw_offset = offset;
nkwq->kw_type = (wqtype & KSYN_WQTYPE_MASK);
nkwq->kw_lastseqword = PTHRW_RWS_INIT;
if (nkwq->kw_type == KSYN_WQTYPE_RWLOCK)
nkwq->kw_nextseqword = PTHRW_RWS_INIT;
nkwq->kw_pre_sseq = PTHRW_RWS_INIT;
CLEAR_PREPOST_BITS(nkwq);
CLEAR_INTR_PREPOST_BITS(nkwq);
CLEAR_REINIT_BITS(nkwq);
nkwq->kw_lword = mgen;
nkwq->kw_uword = ugen;
nkwq->kw_sword = rw_wc;
nkwq->kw_owner = tid;
for (i=0; i< KSYN_QUEUE_MAX; i++)
ksyn_queue_init(&nkwq->kw_ksynqueues[i]);
mutex_init(&nkwq->kw_lock);
//pthread_list_lock_spin();
pthread_list_lock();
/* see whether it is alread allocated */
kwq = ksyn_wq_hash_lookup(mutex, p, flags, object, offset);
if (kwq != NULL) {
if ((kwq->kw_pflags & KSYN_WQ_FLIST) != 0) {
LIST_REMOVE(kwq, kw_list);
kwq->kw_pflags &= ~KSYN_WQ_FLIST;
num_infreekwq--;
num_reusekwq++;
}
if ((kwq->kw_type & KSYN_WQTYPE_MASK) != (wqtype &KSYN_WQTYPE_MASK)) {
if ((kwq->kw_inqueue == 0) && (kwq->kw_pre_rwwc ==0) && (kwq->kw_pre_intrcount == 0)) {
if (kwq->kw_iocount == 0) {
kwq->kw_addr = mutex;
kwq->kw_flags = flags;
kwq->kw_object = object;
kwq->kw_offset = offset;
kwq->kw_type = (wqtype & KSYN_WQTYPE_MASK);
CLEAR_REINIT_BITS(kwq);
CLEAR_INTR_PREPOST_BITS(kwq);
CLEAR_PREPOST_BITS(kwq);
kwq->kw_lword = mgen;
kwq->kw_uword = ugen;
kwq->kw_sword = rw_wc;
kwq->kw_owner = tid;
} else if ((kwq->kw_iocount == 1) && (kwq->kw_dropcount == kwq->kw_iocount)) {
kwq->kw_pflags |= KSYN_WQ_WAITING;
pthread_list_unlock();
/* wait for the wq to be free */
#ifndef __DARLING__
(void)msleep(&kwq->kw_pflags, pthread_list_mlock, PDROP, "ksyn_wqfind", 0);
#else
pthread_list_unlock();
if (wait_event_interruptible(kwq->linux_wq, !(kwq->kw_pflags & KSYN_WQ_WAITING) || darling_thread_canceled()) != 0
|| darling_thread_canceled())
{
return LINUX_EINTR;
}
#endif
// lck_mtx_destroy(&nkwq->kw_lock, pthread_lck_grp);
zfree(kwq_zone, nkwq);
/* will acquire lock again */
goto loop;
} else {
__FAILEDUSERTEST__("address already known to kernel for another [busy] synchronizer type(2)\n");
pthread_list_unlock();
lck_mtx_destroy(&nkwq->kw_lock, pthread_lck_grp);
zfree(kwq_zone, nkwq);
return LINUX_EBUSY;
}
} else {
__FAILEDUSERTEST__("address already known to kernel for another [busy] synchronizer type(3)\n");
pthread_list_unlock();
lck_mtx_destroy(&nkwq->kw_lock, pthread_lck_grp);
zfree(kwq_zone, nkwq);
return LINUX_EBUSY;
}
}
kwq->kw_iocount++;
if (wqtype == KSYN_WQTYPE_MUTEXDROP)
kwq->kw_dropcount++;
if (kwqp != NULL)
*kwqp = kwq;
pthread_list_unlock();
lck_mtx_destroy(&nkwq->kw_lock, pthread_lck_grp);
zfree(kwq_zone, nkwq);
return (0);
}
kwq = nkwq;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVSEQ | DBG_FUNC_NONE, kwq->kw_lword, kwq->kw_uword, kwq->kw_sword, 0xffff, 0);
#endif /* _PSYNCH_TRACE_ */
if ((flags & PTHREAD_PSHARED_FLAGS_MASK) == PTHREAD_PROCESS_SHARED)
{
kwq->kw_pflags |= KSYN_WQ_SHARED;
LIST_INSERT_HEAD(&hashptr[kwq->kw_object & pthhash], kwq, kw_hash);
} else
LIST_INSERT_HEAD(&hashptr[mutex & pthhash], kwq, kw_hash);
kwq->kw_pflags |= KSYN_WQ_INHASH;
num_total_kwq++;
pthread_list_unlock();
if (kwqp != NULL)
*kwqp = kwq;
return (0);
}
/* Reference from find is dropped here. Starts the free process if needed */
void
ksyn_wqrelease(ksyn_wait_queue_t kwq, ksyn_wait_queue_t ckwq, int qfreenow, int wqtype)
{
uint64_t deadline;
struct linux_timespec t;
int sched = 0;
ksyn_wait_queue_t free_elem = NULL;
ksyn_wait_queue_t free_elem1 = NULL;
//pthread_list_lock_spin();
pthread_list_lock();
kwq->kw_iocount--;
if (wqtype == KSYN_WQTYPE_MUTEXDROP) {
kwq->kw_dropcount--;
}
if (kwq->kw_iocount == 0) {
if ((kwq->kw_pflags & KSYN_WQ_WAITING) != 0) {
/* some one is waiting for the waitqueue, wake them up */
kwq->kw_pflags &= ~KSYN_WQ_WAITING;
#ifndef __DARLING__
wakeup(&kwq->kw_pflags);
#else
wake_up_interruptible(&kwq->linux_wq);
#endif
}
if ((kwq->kw_pre_rwwc == 0) && (kwq->kw_inqueue == 0) && (kwq->kw_pre_intrcount == 0)) {
if (qfreenow == 0) {
microuptime(&kwq->kw_ts);
LIST_INSERT_HEAD(&pth_free_list, kwq, kw_list);
kwq->kw_pflags |= KSYN_WQ_FLIST;
num_infreekwq++;
free_elem = NULL;
} else {
/* remove from the only list it is in ie hash */
kwq->kw_pflags &= ~(KSYN_WQ_FLIST | KSYN_WQ_INHASH);
LIST_REMOVE(kwq, kw_hash);
lck_mtx_destroy(&kwq->kw_lock, pthread_lck_grp);
num_total_kwq--;
num_freekwq++;
free_elem = kwq;
}
} else
free_elem = NULL;
if (qfreenow == 0)
sched = 1;
}
if (ckwq != NULL) {
ckwq->kw_iocount--;
if (wqtype == KSYN_WQTYPE_MUTEXDROP) {
kwq->kw_dropcount--;
}
if ( ckwq->kw_iocount == 0) {
if ((kwq->kw_pflags & KSYN_WQ_WAITING) != 0) {
/* some one is waiting for the waitqueue, wake them up */
kwq->kw_pflags &= ~KSYN_WQ_WAITING;
#ifndef __DARLING__
wakeup(&kwq->kw_pflags);
#else
wake_up_interruptible(&kwq->linux_wq);
#endif
}
if ((ckwq->kw_pre_rwwc == 0) && (ckwq->kw_inqueue == 0) && (ckwq->kw_pre_intrcount == 0)) {
if (qfreenow == 0) {
/* mark for free if we can */
microuptime(&ckwq->kw_ts);
LIST_INSERT_HEAD(&pth_free_list, ckwq, kw_list);
ckwq->kw_pflags |= KSYN_WQ_FLIST;
num_infreekwq++;
free_elem1 = NULL;
} else {
/* remove from the only list it is in ie hash */
ckwq->kw_pflags &= ~(KSYN_WQ_FLIST | KSYN_WQ_INHASH);
LIST_REMOVE(ckwq, kw_hash);
lck_mtx_destroy(&ckwq->kw_lock, pthread_lck_grp);
num_total_kwq--;
num_freekwq++;
free_elem1 = ckwq;
}
} else
free_elem1 = NULL;
if (qfreenow == 0)
sched = 1;
}
}
if (sched == 1 && psynch_cleanupset == 0) {
psynch_cleanupset = 1;
microuptime(&t);
t.tv_sec += KSYN_CLEANUP_DEADLINE;
deadline = tvtoabstime(&t);
thread_call_enter_delayed(psynch_thcall, deadline);
}
pthread_list_unlock();
if (free_elem != NULL)
zfree(kwq_zone, free_elem);
if (free_elem1 != NULL)
zfree(kwq_zone, free_elem1);
}
/* responsible to free the waitqueues */
void
psynch_wq_cleanup(__unused void * param, __unused void * param1)
{
ksyn_wait_queue_t kwq;
struct linux_timespec t;
LIST_HEAD(freelist);
int count = 0, delayed = 0, diff;
uint64_t deadline = 0;
//pthread_list_lock_spin();
pthread_list_lock();
num_addedfreekwq = num_infreekwq - num_lastfreekwqcount;
num_lastfreekwqcount = num_infreekwq;
microuptime(&t);
LIST_FOREACH(kwq, &pth_free_list, kw_list) {
if ((kwq->kw_iocount != 0) || (kwq->kw_pre_rwwc != 0) || (kwq->kw_inqueue != 0) || (kwq->kw_pre_intrcount != 0)) {
/* still in use */
continue;
}
diff = t.tv_sec - kwq->kw_ts.tv_sec;
if (diff < 0)
diff *= -1;
if (diff >= KSYN_CLEANUP_DEADLINE) {
/* out of hash */
kwq->kw_pflags &= ~(KSYN_WQ_FLIST | KSYN_WQ_INHASH);
num_infreekwq--;
num_freekwq++;
LIST_REMOVE(kwq, kw_hash);
LIST_REMOVE(kwq, kw_list);
LIST_INSERT_HEAD(&freelist, kwq, kw_list);
count ++;
num_total_kwq--;
} else {
delayed = 1;
}
}
if (delayed != 0) {
t.tv_sec += KSYN_CLEANUP_DEADLINE;
deadline = tvtoabstime(&t);
thread_call_enter_delayed(psynch_thcall, deadline);
psynch_cleanupset = 1;
} else
psynch_cleanupset = 0;
pthread_list_unlock();
while ((kwq = LIST_FIRST(&freelist, kw_list)) != NULL) {
LIST_REMOVE(kwq, kw_list);
#ifndef __DARLING__
lck_mtx_destroy(&kwq->kw_lock, pthread_lck_grp);
#endif
zfree(kwq_zone, kwq);
}
}
kern_return_t
#if _PSYNCH_TRACE_
ksyn_block_thread_locked(ksyn_wait_queue_t kwq, uint64_t abstime, ksyn_waitq_element_t kwe, int mylog, thread_continue_t continuation, void * parameter)
#else
ksyn_block_thread_locked(ksyn_wait_queue_t kwq, uint64_t abstime, ksyn_waitq_element_t kwe, __unused int mylog, thread_continue_t continuation, void * parameter)
#endif
{
kern_return_t kret;
#if _PSYNCH_TRACE_
int error = 0;
uthread_t uth = NULL;
#endif /* _PSYNCH_TRACE_ */
kwe->kwe_kwqqueue = (void *)kwq;
assert_wait_deadline(&kwe->kwe_psynchretval, THREAD_ABORTSAFE, abstime);
ksyn_wqunlock(kwq);
if (continuation == THREAD_CONTINUE_NULL)
kret = thread_block(NULL);
else
kret = thread_block_parameter(continuation, parameter);
#if _PSYNCH_TRACE_
switch (kret) {
case THREAD_TIMED_OUT:
error = ETIMEDOUT;
break;
case THREAD_INTERRUPTED:
error = EINTR;
break;
}
uth = current_uthread();
#if defined(__i386__)
if (mylog != 0)
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_THWAKEUP | DBG_FUNC_NONE, 0xf4f3f2f1, (uint32_t)uth, kret, 0, 0);
#else
if (mylog != 0)
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_THWAKEUP | DBG_FUNC_NONE, 0xeeeeeeee, kret, error, 0xeeeeeeee, 0);
#endif
#endif /* _PSYNCH_TRACE_ */
return(kret);
}
kern_return_t
ksyn_wakeup_thread(__unused ksyn_wait_queue_t kwq, ksyn_waitq_element_t kwe)
{
kern_return_t kret;
#if _PSYNCH_TRACE_
uthread_t uth = NULL;
#endif /* _PSYNCH_TRACE_ */
kret = thread_wakeup_one((caddr_t)&kwe->kwe_psynchretval);
if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
panic("ksyn_wakeup_thread: panic waking up thread %x\n", kret);
#if _PSYNCH_TRACE_
uth = kwe->kwe_uth;
#if defined(__i386__)
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_THWAKEUP | DBG_FUNC_NONE, 0xf1f2f3f4, (uint32_t)uth, kret, 0, 0);
#endif
#endif /* _PSYNCH_TRACE_ */
return(kret);
}
/* find the true shared obect/offset for shared mutexes */
int
ksyn_findobj(uint64_t mutex, uint64_t * objectp, uint64_t * offsetp)
{
#ifndef __DARLING__
vm_page_info_basic_data_t info;
kern_return_t kret;
mach_msg_type_number_t count = VM_PAGE_INFO_BASIC_COUNT;
kret = vm_map_page_info(current_map(), mutex, VM_PAGE_INFO_BASIC,
(vm_page_info_t)&info, &count);
if (kret != KERN_SUCCESS)
return(EINVAL);
if (objectp != NULL)
*objectp = (uint64_t)info.object_id;
if (offsetp != NULL)
*offsetp = (uint64_t)info.offset;
return(0);
#else
return LINUX_ENOTSUPP; // TODO
#endif
}
/* lowest of kw_fr, kw_flr, kw_fwr, kw_fywr */
int
kwq_find_rw_lowest(ksyn_wait_queue_t kwq, int flags, uint32_t premgen, int * typep, uint32_t lowest[])
{
uint32_t kw_fr, kw_flr, kw_fwr, kw_fywr, low;
int type = 0, lowtype, typenum[4];
uint32_t numbers[4];
int count = 0, i;
if ((kwq->kw_ksynqueues[KSYN_QUEUE_READ].ksynq_count != 0) || ((flags & KW_UNLOCK_PREPOST_READLOCK) != 0)) {
type |= PTH_RWSHFT_TYPE_READ;
/* read entries are present */
if (kwq->kw_ksynqueues[KSYN_QUEUE_READ].ksynq_count != 0) {
kw_fr = kwq->kw_ksynqueues[KSYN_QUEUE_READ].ksynq_firstnum;
if (((flags & KW_UNLOCK_PREPOST_READLOCK) != 0) && (is_seqlower(premgen, kw_fr) != 0))
kw_fr = premgen;
} else
kw_fr = premgen;
lowest[KSYN_QUEUE_READ] = kw_fr;
numbers[count]= kw_fr;
typenum[count] = PTH_RW_TYPE_READ;
count++;
} else
lowest[KSYN_QUEUE_READ] = 0;
if ((kwq->kw_ksynqueues[KSYN_QUEUE_LREAD].ksynq_count != 0) || ((flags & KW_UNLOCK_PREPOST_LREADLOCK) != 0)) {
type |= PTH_RWSHFT_TYPE_LREAD;
/* read entries are present */
if (kwq->kw_ksynqueues[KSYN_QUEUE_LREAD].ksynq_count != 0) {
kw_flr = kwq->kw_ksynqueues[KSYN_QUEUE_LREAD].ksynq_firstnum;
if (((flags & KW_UNLOCK_PREPOST_LREADLOCK) != 0) && (is_seqlower(premgen, kw_flr) != 0))
kw_flr = premgen;
} else
kw_flr = premgen;
lowest[KSYN_QUEUE_LREAD] = kw_flr;
numbers[count]= kw_flr;
typenum[count] = PTH_RW_TYPE_LREAD;
count++;
} else
lowest[KSYN_QUEUE_LREAD] = 0;
if ((kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_count != 0) || ((flags & KW_UNLOCK_PREPOST_WRLOCK) != 0)) {
type |= PTH_RWSHFT_TYPE_WRITE;
/* read entries are present */
if (kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_count != 0) {
kw_fwr = kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_firstnum;
if (((flags & KW_UNLOCK_PREPOST_WRLOCK) != 0) && (is_seqlower(premgen, kw_fwr) != 0))
kw_fwr = premgen;
} else
kw_fwr = premgen;
lowest[KSYN_QUEUE_WRITER] = kw_fwr;
numbers[count]= kw_fwr;
typenum[count] = PTH_RW_TYPE_WRITE;
count++;
} else
lowest[KSYN_QUEUE_WRITER] = 0;
if ((kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER].ksynq_count != 0) || ((flags & KW_UNLOCK_PREPOST_YWRLOCK) != 0)) {
type |= PTH_RWSHFT_TYPE_YWRITE;
/* read entries are present */
if (kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER].ksynq_count != 0) {
kw_fywr = kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER].ksynq_firstnum;
if (((flags & KW_UNLOCK_PREPOST_YWRLOCK) != 0) && (is_seqlower(premgen, kw_fywr) != 0))
kw_fywr = premgen;
} else
kw_fywr = premgen;
lowest[KSYN_QUEUE_YWRITER] = kw_fywr;
numbers[count]= kw_fywr;
typenum[count] = PTH_RW_TYPE_YWRITE;
count++;
} else
lowest[KSYN_QUEUE_YWRITER] = 0;
#if __TESTPANICS__
if (count == 0)
panic("nothing in the queue???\n");
#endif /* __TESTPANICS__ */
low = numbers[0];
lowtype = typenum[0];
if (count > 1) {
for (i = 1; i< count; i++) {
if(is_seqlower(numbers[i] , low) != 0) {
low = numbers[i];
lowtype = typenum[i];
}
}
}
type |= lowtype;
if (typep != 0)
*typep = type;
return(0);
}
/* wakeup readers and longreaders to upto the writer limits */
int
ksyn_wakeupreaders(ksyn_wait_queue_t kwq, uint32_t limitread, int longreadset, int allreaders, uint32_t updatebits, int * wokenp)
{
ksyn_waitq_element_t kwe = NULL;
ksyn_queue_t kq;
int failedwakeup = 0;
int numwoken = 0;
kern_return_t kret = KERN_SUCCESS;
uint32_t lbits = 0;
lbits = updatebits;
if (longreadset != 0) {
/* clear all read and longreads */
while ((kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_READ], kwq)) != NULL) {
kwe->kwe_psynchretval = lbits;
kwe->kwe_kwqqueue = NULL;
numwoken++;
kret = ksyn_wakeup_thread(kwq, kwe);
#if __TESTPANICS__
if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
panic("ksyn_wakeupreaders: panic waking up readers\n");
#endif /* __TESTPANICS__ */
if (kret == KERN_NOT_WAITING) {
failedwakeup++;
}
}
while ((kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_LREAD], kwq)) != NULL) {
kwe->kwe_psynchretval = lbits;
kwe->kwe_kwqqueue = NULL;
numwoken++;
kret = ksyn_wakeup_thread(kwq, kwe);
#if __TESTPANICS__
if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
panic("ksyn_wakeupreaders: panic waking up lreaders\n");
#endif /* __TESTPANICS__ */
if (kret == KERN_NOT_WAITING) {
failedwakeup++;
}
}
} else {
kq = &kwq->kw_ksynqueues[KSYN_QUEUE_READ];
while ((kq->ksynq_count != 0) && (allreaders || (is_seqlower(kq->ksynq_firstnum, limitread) != 0))) {
kwe = ksyn_queue_removefirst(kq, kwq);
kwe->kwe_psynchretval = lbits;
kwe->kwe_kwqqueue = NULL;
numwoken++;
kret = ksyn_wakeup_thread(kwq, kwe);
#if __TESTPANICS__
if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
panic("ksyn_wakeupreaders: panic waking up readers\n");
#endif /* __TESTPANICS__ */
if (kret == KERN_NOT_WAITING) {
failedwakeup++;
}
}
}
if (wokenp != NULL)
*wokenp = numwoken;
return(failedwakeup);
}
/* This handles the unlock grants for next set on rw_unlock() or on arrival of all preposted waiters */
int
kwq_handle_unlock(ksyn_wait_queue_t kwq, uint32_t mgen, uint32_t rw_wc, uint32_t * updatep, int flags, int * blockp, uint32_t premgen)
{
uint32_t low_reader, low_writer, low_ywriter, low_lreader,limitrdnum;
int rwtype, error=0;
int longreadset = 0, allreaders, failed;
uint32_t updatebits=0, numneeded = 0;;
int prepost = flags & KW_UNLOCK_PREPOST;
thread_t preth = THREAD_NULL;
ksyn_waitq_element_t kwe;
uthread_t uth;
thread_t th;
int woken = 0;
int block = 1;
uint32_t lowest[KSYN_QUEUE_MAX]; /* np need for upgrade as it is handled separately */
kern_return_t kret = KERN_SUCCESS;
ksyn_queue_t kq;
int curthreturns = 0;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWHANDLEU | DBG_FUNC_START, (uint32_t)kwq->kw_addr, mgen, premgen, rw_wc, 0);
#endif /* _PSYNCH_TRACE_ */
if (prepost != 0) {
preth = current_thread();
}
kq = &kwq->kw_ksynqueues[KSYN_QUEUE_READ];
kwq->kw_lastseqword = rw_wc;
kwq->kw_lastunlockseq = (rw_wc & PTHRW_COUNT_MASK);
kwq->kw_overlapwatch = 0;
/* upgrade pending */
if (is_rw_ubit_set(mgen)) {
#if __TESTPANICS__
panic("NO UBIT SHOULD BE SET\n");
updatebits = PTH_RWL_EBIT | PTH_RWL_KBIT;
if (kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_count != 0)
updatebits |= PTH_RWL_WBIT;
if (kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER].ksynq_count != 0)
updatebits |= PTH_RWL_YBIT;
if (prepost != 0) {
if((flags & KW_UNLOCK_PREPOST_UPGRADE) != 0) {
/* upgrade thread calling the prepost */
/* upgrade granted */
block = 0;
goto out;
}
}
if (kwq->kw_ksynqueues[KSYN_QUEUE_UPGRADE].ksynq_count > 0) {
kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_UPGRADE], kwq);
kwq->kw_nextseqword = (rw_wc & PTHRW_COUNT_MASK) + updatebits;
kwe->kwe_psynchretval = updatebits;
kwe->kwe_kwqqueue = NULL;
kret = ksyn_wakeup_thread(kwq, kwe);
if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
panic("kwq_handle_unlock: panic waking up the upgrade thread \n");
if (kret == KERN_NOT_WAITING) {
kwq->kw_pre_intrcount = 1; /* actually a count */
kwq->kw_pre_intrseq = mgen;
kwq->kw_pre_intrretbits = kwe->kwe_psynchretval;
kwq->kw_pre_intrtype = PTH_RW_TYPE_UPGRADE;
}
error = 0;
} else {
panic("panic unable to find the upgrade thread\n");
}
#endif /* __TESTPANICS__ */
ksyn_wqunlock(kwq);
goto out;
}
error = kwq_find_rw_lowest(kwq, flags, premgen, &rwtype, lowest);
#if __TESTPANICS__
if (error != 0)
panic("rwunlock: cannot fails to slot next round of threads");
#endif /* __TESTPANICS__ */
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWHANDLEU | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 1, rwtype, 0, 0);
#endif /* _PSYNCH_TRACE_ */
low_reader = lowest[KSYN_QUEUE_READ];
low_lreader = lowest[KSYN_QUEUE_LREAD];
low_writer = lowest[KSYN_QUEUE_WRITER];
low_ywriter = lowest[KSYN_QUEUE_YWRITER];
longreadset = 0;
allreaders = 0;
updatebits = 0;
switch (rwtype & PTH_RW_TYPE_MASK) {
case PTH_RW_TYPE_LREAD:
longreadset = 1;
case PTH_RW_TYPE_READ: {
/* what about the preflight which is LREAD or READ ?? */
if ((rwtype & PTH_RWSHFT_TYPE_MASK) != 0) {
if (rwtype & PTH_RWSHFT_TYPE_WRITE)
updatebits |= (PTH_RWL_WBIT | PTH_RWL_KBIT);
if (rwtype & PTH_RWSHFT_TYPE_YWRITE)
updatebits |= PTH_RWL_YBIT;
}
limitrdnum = 0;
if (longreadset == 0) {
switch (rwtype & (PTH_RWSHFT_TYPE_WRITE | PTH_RWSHFT_TYPE_YWRITE)) {
case PTH_RWSHFT_TYPE_WRITE:
limitrdnum = low_writer;
if (((rwtype & PTH_RWSHFT_TYPE_LREAD) != 0) &&
(is_seqlower(low_lreader, limitrdnum) != 0)) {
longreadset = 1;
}
if (((flags & KW_UNLOCK_PREPOST_LREADLOCK) != 0) &&
(is_seqlower(premgen, limitrdnum) != 0)) {
longreadset = 1;
}
break;
case PTH_RWSHFT_TYPE_YWRITE:
/* all read ? */
if (((rwtype & PTH_RWSHFT_TYPE_LREAD) != 0) &&
(is_seqlower(low_lreader, low_ywriter) != 0)) {
longreadset = 1;
} else
allreaders = 1;
if (((flags & KW_UNLOCK_PREPOST_LREADLOCK) != 0) &&
(is_seqlower(premgen, low_ywriter) != 0)) {
longreadset = 1;
allreaders = 0;
}
break;
case (PTH_RWSHFT_TYPE_WRITE | PTH_RWSHFT_TYPE_YWRITE):
if (is_seqlower(low_ywriter, low_writer) != 0) {
limitrdnum = low_ywriter;
} else
limitrdnum = low_writer;
if (((rwtype & PTH_RWSHFT_TYPE_LREAD) != 0) &&
(is_seqlower(low_lreader, limitrdnum) != 0)) {
longreadset = 1;
}
if (((flags & KW_UNLOCK_PREPOST_LREADLOCK) != 0) &&
(is_seqlower(premgen, limitrdnum) != 0)) {
longreadset = 1;
}
break;
default: /* no writers at all */
if ((rwtype & PTH_RWSHFT_TYPE_LREAD) != 0)
longreadset = 1;
else
allreaders = 1;
};
}
numneeded = 0;
if (longreadset != 0) {
updatebits |= PTH_RWL_LBIT;
updatebits &= ~PTH_RWL_KBIT;
if ((flags & (KW_UNLOCK_PREPOST_READLOCK | KW_UNLOCK_PREPOST_LREADLOCK)) != 0)
numneeded += 1;
numneeded += kwq->kw_ksynqueues[KSYN_QUEUE_READ].ksynq_count;
numneeded += kwq->kw_ksynqueues[KSYN_QUEUE_LREAD].ksynq_count;
updatebits += (numneeded << PTHRW_COUNT_SHIFT);
kwq->kw_overlapwatch = 1;
} else {
/* no longread, evaluate number of readers */
switch (rwtype & (PTH_RWSHFT_TYPE_WRITE | PTH_RWSHFT_TYPE_YWRITE)) {
case PTH_RWSHFT_TYPE_WRITE:
limitrdnum = low_writer;
numneeded = ksyn_queue_count_tolowest(kq, limitrdnum);
if (((flags & KW_UNLOCK_PREPOST_READLOCK) != 0) && (is_seqlower(premgen, limitrdnum) != 0)) {
curthreturns = 1;
numneeded += 1;
}
break;
case PTH_RWSHFT_TYPE_YWRITE:
/* all read ? */
numneeded += kwq->kw_ksynqueues[KSYN_QUEUE_READ].ksynq_count;
if ((flags & KW_UNLOCK_PREPOST_READLOCK) != 0) {
curthreturns = 1;
numneeded += 1;
}
break;
case (PTH_RWSHFT_TYPE_WRITE | PTH_RWSHFT_TYPE_YWRITE):
limitrdnum = low_writer;
numneeded = ksyn_queue_count_tolowest(kq, limitrdnum);
if (((flags & KW_UNLOCK_PREPOST_READLOCK) != 0) && (is_seqlower(premgen, limitrdnum) != 0)) {
curthreturns = 1;
numneeded += 1;
}
break;
default: /* no writers at all */
/* no other waiters only readers */
kwq->kw_overlapwatch = 1;
numneeded += kwq->kw_ksynqueues[KSYN_QUEUE_READ].ksynq_count;
if ((flags & KW_UNLOCK_PREPOST_READLOCK) != 0) {
curthreturns = 1;
numneeded += 1;
}
};
updatebits += (numneeded << PTHRW_COUNT_SHIFT);
}
kwq->kw_nextseqword = (rw_wc & PTHRW_COUNT_MASK) + updatebits;
if (curthreturns != 0) {
block = 0;
uth = current_uthread();
kwe = &uth->uu_save.uus_kwe;
kwe->kwe_psynchretval = updatebits;
}
failed = ksyn_wakeupreaders(kwq, limitrdnum, longreadset, allreaders, updatebits, &woken);
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWHANDLEU | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 2, woken, failed, 0);
#endif /* _PSYNCH_TRACE_ */
if (failed != 0) {
kwq->kw_pre_intrcount = failed; /* actually a count */
kwq->kw_pre_intrseq = limitrdnum;
kwq->kw_pre_intrretbits = updatebits;
if (longreadset)
kwq->kw_pre_intrtype = PTH_RW_TYPE_LREAD;
else
kwq->kw_pre_intrtype = PTH_RW_TYPE_READ;
}
error = 0;
if ((kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_count != 0) && ((updatebits & PTH_RWL_WBIT) == 0))
panic("kwq_handle_unlock: writer pending but no writebit set %x\n", updatebits);
}
break;
case PTH_RW_TYPE_WRITE: {
/* only one thread is goin to be granted */
updatebits |= (PTHRW_INC);
updatebits |= PTH_RWL_KBIT| PTH_RWL_EBIT;
if (((flags & KW_UNLOCK_PREPOST_WRLOCK) != 0) && (low_writer == premgen)) {
block = 0;
if (kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_count != 0)
updatebits |= PTH_RWL_WBIT;
if ((rwtype & PTH_RWSHFT_TYPE_YWRITE) != 0)
updatebits |= PTH_RWL_YBIT;
th = preth;
uth = get_bsdthread_info(th);
kwe = &uth->uu_save.uus_kwe;
kwe->kwe_psynchretval = updatebits;
} else {
/* we are not granting writelock to the preposting thread */
kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_WRITER], kwq);
/* if there are writers present or the preposting write thread then W bit is to be set */
if ((kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_count != 0) || ((flags & KW_UNLOCK_PREPOST_WRLOCK) != 0) )
updatebits |= PTH_RWL_WBIT;
if ((rwtype & PTH_RWSHFT_TYPE_YWRITE) != 0)
updatebits |= PTH_RWL_YBIT;
kwe->kwe_psynchretval = updatebits;
kwe->kwe_kwqqueue = NULL;
/* setup next in the queue */
kret = ksyn_wakeup_thread(kwq, kwe);
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWHANDLEU | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 3, kret, 0, 0);
#endif /* _PSYNCH_TRACE_ */
#if __TESTPANICS__
if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
panic("kwq_handle_unlock: panic waking up writer\n");
#endif /* __TESTPANICS__ */
if (kret == KERN_NOT_WAITING) {
kwq->kw_pre_intrcount = 1; /* actually a count */
kwq->kw_pre_intrseq = low_writer;
kwq->kw_pre_intrretbits = updatebits;
kwq->kw_pre_intrtype = PTH_RW_TYPE_WRITE;
}
error = 0;
}
kwq->kw_nextseqword = (rw_wc & PTHRW_COUNT_MASK) + updatebits;
if ((updatebits & (PTH_RWL_KBIT | PTH_RWL_EBIT)) != (PTH_RWL_KBIT | PTH_RWL_EBIT))
panic("kwq_handle_unlock: writer lock granted but no ke set %x\n", updatebits);
}
break;
case PTH_RW_TYPE_YWRITE: {
/* can reader locks be granted ahead of this write? */
if ((rwtype & PTH_RWSHFT_TYPE_READ) != 0) {
if ((rwtype & PTH_RWSHFT_TYPE_MASK) != 0) {
if (rwtype & PTH_RWSHFT_TYPE_WRITE)
updatebits |= (PTH_RWL_WBIT | PTH_RWL_KBIT);
if (rwtype & PTH_RWSHFT_TYPE_YWRITE)
updatebits |= PTH_RWL_YBIT;
}
if ((rwtype & PTH_RWSHFT_TYPE_WRITE) != 0) {
/* is lowest reader less than the low writer? */
if (is_seqlower(low_reader,low_writer) == 0)
goto yielditis;
numneeded = ksyn_queue_count_tolowest(kq, low_writer);
updatebits += (numneeded << PTHRW_COUNT_SHIFT);
if (((flags & KW_UNLOCK_PREPOST_READLOCK) != 0) && (is_seqlower(premgen, low_writer) != 0)) {
uth = current_uthread();
kwe = &uth->uu_save.uus_kwe;
/* add one more */
updatebits += PTHRW_INC;
kwe->kwe_psynchretval = updatebits;
block = 0;
}
kwq->kw_nextseqword = (rw_wc & PTHRW_COUNT_MASK) + updatebits;
/* there will be readers to wakeup , no need to check for woken */
failed = ksyn_wakeupreaders(kwq, low_writer, 0, 0, updatebits, NULL);
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWHANDLEU | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 2, woken, failed, 0);
#endif /* _PSYNCH_TRACE_ */
if (failed != 0) {
kwq->kw_pre_intrcount = failed; /* actually a count */
kwq->kw_pre_intrseq = low_writer;
kwq->kw_pre_intrretbits = updatebits;
kwq->kw_pre_intrtype = PTH_RW_TYPE_READ;
}
error = 0;
} else {
/* wakeup all readers */
numneeded = kwq->kw_ksynqueues[KSYN_QUEUE_READ].ksynq_count;
updatebits += (numneeded << PTHRW_COUNT_SHIFT);
if ((prepost != 0) && ((flags & KW_UNLOCK_PREPOST_READLOCK) != 0)) {
uth = current_uthread();
kwe = &uth->uu_save.uus_kwe;
updatebits += PTHRW_INC;
kwe->kwe_psynchretval = updatebits;
block = 0;
}
kwq->kw_nextseqword = (rw_wc & PTHRW_COUNT_MASK) + updatebits;
failed = ksyn_wakeupreaders(kwq, low_writer, 0, 1, updatebits, &woken);
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWHANDLEU | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 2, woken, failed, 0);
#endif /* _PSYNCH_TRACE_ */
if (failed != 0) {
kwq->kw_pre_intrcount = failed; /* actually a count */
kwq->kw_pre_intrseq = kwq->kw_highseq;
kwq->kw_pre_intrretbits = updatebits;
kwq->kw_pre_intrtype = PTH_RW_TYPE_READ;
}
error = 0;
}
} else {
yielditis:
/* no reads, so granting yeilding writes */
updatebits |= PTHRW_INC;
updatebits |= PTH_RWL_KBIT| PTH_RWL_EBIT;
if (((flags & KW_UNLOCK_PREPOST_YWRLOCK) != 0) && (low_writer == premgen)) {
/* preposting yielding write thread is being granted exclusive lock */
block = 0;
if ((rwtype & PTH_RWSHFT_TYPE_WRITE) != 0)
updatebits |= PTH_RWL_WBIT;
else if (kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER].ksynq_count != 0)
updatebits |= PTH_RWL_YBIT;
th = preth;
uth = get_bsdthread_info(th);
kwe = &uth->uu_save.uus_kwe;
kwe->kwe_psynchretval = updatebits;
} else {
/* we are granting yield writelock to some other thread */
kwe = ksyn_queue_removefirst(&kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER], kwq);
if ((rwtype & PTH_RWSHFT_TYPE_WRITE) != 0)
updatebits |= PTH_RWL_WBIT;
/* if there are ywriters present or the preposting ywrite thread then W bit is to be set */
else if ((kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER].ksynq_count != 0) || ((flags & KW_UNLOCK_PREPOST_YWRLOCK) != 0) )
updatebits |= PTH_RWL_YBIT;
kwe->kwe_psynchretval = updatebits;
kwe->kwe_kwqqueue = NULL;
kret = ksyn_wakeup_thread(kwq, kwe);
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWHANDLEU | DBG_FUNC_NONE, (uint32_t)kwq->kw_addr, 3, kret, 0, 0);
#endif /* _PSYNCH_TRACE_ */
#if __TESTPANICS__
if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
panic("kwq_handle_unlock : panic waking up readers\n");
#endif /* __TESTPANICS__ */
if (kret == KERN_NOT_WAITING) {
kwq->kw_pre_intrcount = 1; /* actually a count */
kwq->kw_pre_intrseq = low_ywriter;
kwq->kw_pre_intrretbits = updatebits;
kwq->kw_pre_intrtype = PTH_RW_TYPE_YWRITE;
}
error = 0;
}
kwq->kw_nextseqword = (rw_wc & PTHRW_COUNT_MASK) + updatebits;
}
}
break;
default:
panic("rwunlock: invalid type for lock grants");
};
out:
if (updatep != NULL)
*updatep = updatebits;
if (blockp != NULL)
*blockp = block;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_RWHANDLEU | DBG_FUNC_END, (uint32_t)kwq->kw_addr, 0, updatebits, block, 0);
#endif /* _PSYNCH_TRACE_ */
return(error);
}
int
kwq_handle_overlap(ksyn_wait_queue_t kwq, uint32_t lgenval, __unused uint32_t ugenval, uint32_t rw_wc, uint32_t *updatebitsp, __unused int flags , int * blockp)
{
uint32_t highword = kwq->kw_nextseqword & PTHRW_COUNT_MASK;
uint32_t lowword = kwq->kw_lastseqword & PTHRW_COUNT_MASK;
uint32_t val=0;
int withinseq;
/* overlap is set, so no need to check for valid state for overlap */
withinseq = ((is_seqlower_eq(rw_wc, highword) != 0) || (is_seqhigher_eq(lowword, rw_wc) != 0));
if (withinseq != 0) {
if ((kwq->kw_nextseqword & PTH_RWL_LBIT) == 0) {
/* if no writers ahead, overlap granted */
if ((lgenval & PTH_RWL_WBIT) == 0) {
goto grantoverlap;
}
} else {
/* Lbit is set, and writers ahead does not count */
goto grantoverlap;
}
}
*blockp = 1;
return(0);
grantoverlap:
/* increase the next expected seq by one */
kwq->kw_nextseqword += PTHRW_INC;
/* set count by one & bits from the nextseq and add M bit */
val = PTHRW_INC;
val |= ((kwq->kw_nextseqword & PTHRW_BIT_MASK) | PTH_RWL_MBIT);
*updatebitsp = val;
*blockp = 0;
return(0);
}
#if NOTYET
/* handle downgrade actions */
int
kwq_handle_downgrade(ksyn_wait_queue_t kwq, uint32_t mgen, __unused int flags, __unused uint32_t premgen, __unused int * blockp)
{
uint32_t updatebits, lowriter = 0;
int longreadset, allreaders, count;
/* can handle downgrade now */
updatebits = mgen;
longreadset = 0;
allreaders = 0;
if (kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_count > 0) {
lowriter = kwq->kw_ksynqueues[KSYN_QUEUE_WRITER].ksynq_firstnum;
if (kwq->kw_ksynqueues[KSYN_QUEUE_LREAD].ksynq_count > 0) {
if (is_seqlower(kwq->kw_ksynqueues[KSYN_QUEUE_LREAD].ksynq_firstnum, lowriter) != 0)
longreadset = 1;
}
} else {
allreaders = 1;
if (kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER].ksynq_count > 0) {
lowriter = kwq->kw_ksynqueues[KSYN_QUEUE_YWRITER].ksynq_firstnum;
if (kwq->kw_ksynqueues[KSYN_QUEUE_LREAD].ksynq_count > 0) {
if (is_seqlower(kwq->kw_ksynqueues[KSYN_QUEUE_LREAD].ksynq_firstnum, lowriter) != 0)
longreadset = 1;
}
}
}
count = ksyn_wakeupreaders(kwq, lowriter, longreadset, allreaders, updatebits, NULL);
if (count != 0) {
kwq->kw_pre_limrd = count;
kwq->kw_pre_limrdseq = lowriter;
kwq->kw_pre_limrdbits = lowriter;
/* need to handle prepost */
}
return(0);
}
#endif /* NOTYET */
/************* Indiv queue support routines ************************/
void
ksyn_queue_init(ksyn_queue_t kq)
{
INIT_LIST_HEAD(&kq->ksynq_kwelist);
kq->ksynq_count = 0;
kq->ksynq_firstnum = 0;
kq->ksynq_lastnum = 0;
}
int
ksyn_queue_insert(ksyn_wait_queue_t kwq, ksyn_queue_t kq, uint32_t mgen, uthread_t uth, ksyn_waitq_element_t kwe, int fit)
{
uint32_t lockseq = mgen & PTHRW_COUNT_MASK;
ksyn_waitq_element_t q_kwe, r_kwe;
int res = 0;
uthread_t nuth = NULL;
if (kq->ksynq_count == 0) {
TAILQ_INSERT_HEAD(&kq->ksynq_kwelist, kwe, kwe_list);
kq->ksynq_firstnum = lockseq;
kq->ksynq_lastnum = lockseq;
goto out;
}
if (fit == FIRSTFIT) {
/* TBD: if retry bit is set for mutex, add it to the head */
/* firstfit, arriving order */
TAILQ_INSERT_TAIL(&kq->ksynq_kwelist, kwe, kwe_list);
if (is_seqlower (lockseq, kq->ksynq_firstnum) != 0)
kq->ksynq_firstnum = lockseq;
if (is_seqhigher (lockseq, kq->ksynq_lastnum) != 0)
kq->ksynq_lastnum = lockseq;
goto out;
}
#ifndef __DARLING__
if ((lockseq == kq->ksynq_firstnum) || (lockseq == kq->ksynq_lastnum)) {
/* During prepost when a thread is getting cancelled, we could have two with same seq */
if (kwe->kwe_flags == KWE_THREAD_PREPOST) {
q_kwe = ksyn_queue_find_seq(kwq, kq, lockseq, 0);
if ((q_kwe != NULL) && ((nuth = (uthread_t)q_kwe->kwe_uth) != NULL) &&
((nuth->uu_flag & (UT_CANCELDISABLE | UT_CANCEL | UT_CANCELED)) == UT_CANCEL)) {
TAILQ_INSERT_TAIL(&kq->ksynq_kwelist, kwe, kwe_list);
goto out;
} else {
__FAILEDUSERTEST__("ksyn_queue_insert: two threads with same lockseq ");
res = EBUSY;
goto out1;
}
} else {
__FAILEDUSERTEST__("ksyn_queue_insert: two threads with same lockseq ");
res = EBUSY;
goto out1;
}
}
#endif
/* check for next seq one */
if (is_seqlower(kq->ksynq_lastnum, lockseq) != 0) {
TAILQ_INSERT_TAIL(&kq->ksynq_kwelist, kwe, kwe_list);
kq->ksynq_lastnum = lockseq;
goto out;
}
if (is_seqlower(lockseq, kq->ksynq_firstnum) != 0) {
TAILQ_INSERT_HEAD(&kq->ksynq_kwelist, kwe, kwe_list);
kq->ksynq_firstnum = lockseq;
goto out;
}
/* goto slow insert mode */
TAILQ_FOREACH_SAFE(q_kwe, &kq->ksynq_kwelist, kwe_list, r_kwe) {
if (is_seqhigher(q_kwe->kwe_lockseq, lockseq) != 0) {
TAILQ_INSERT_BEFORE(q_kwe, kwe, kwe_list);
goto out;
}
}
#if __TESTPANICS__
panic("failed to insert \n");
#endif /* __TESTPANICS__ */
out:
if (uth != NULL)
kwe->kwe_uth = uth;
kq->ksynq_count++;
kwq->kw_inqueue++;
update_low_high(kwq, lockseq);
#ifndef __DARLING__
out1:
#endif
return(res);
}
ksyn_waitq_element_t
ksyn_queue_removefirst(ksyn_queue_t kq, ksyn_wait_queue_t kwq)
{
ksyn_waitq_element_t kwe = NULL;
ksyn_waitq_element_t q_kwe;
uint32_t curseq;
if (kq->ksynq_count != 0) {
kwe = TAILQ_FIRST(&kq->ksynq_kwelist);
TAILQ_REMOVE(&kq->ksynq_kwelist, kwe, kwe_list);
curseq = kwe->kwe_lockseq & PTHRW_COUNT_MASK;
kq->ksynq_count--;
kwq->kw_inqueue--;
if(kq->ksynq_count != 0) {
q_kwe = TAILQ_FIRST(&kq->ksynq_kwelist);
kq->ksynq_firstnum = (q_kwe->kwe_lockseq & PTHRW_COUNT_MASK);
} else {
kq->ksynq_firstnum = 0;
kq->ksynq_lastnum = 0;
}
if (kwq->kw_inqueue == 0) {
kwq->kw_lowseq = 0;
kwq->kw_highseq = 0;
} else {
if (kwq->kw_lowseq == curseq)
kwq->kw_lowseq = find_nextlowseq(kwq);
if (kwq->kw_highseq == curseq)
kwq->kw_highseq = find_nexthighseq(kwq);
}
}
return(kwe);
}
void
ksyn_queue_removeitem(ksyn_wait_queue_t kwq, ksyn_queue_t kq, ksyn_waitq_element_t kwe)
{
ksyn_waitq_element_t q_kwe;
uint32_t curseq;
if (kq->ksynq_count > 0) {
TAILQ_REMOVE(&kq->ksynq_kwelist, kwe, kwe_list);
kq->ksynq_count--;
if(kq->ksynq_count != 0) {
q_kwe = TAILQ_FIRST(&kq->ksynq_kwelist);
kq->ksynq_firstnum = (q_kwe->kwe_lockseq & PTHRW_COUNT_MASK);
q_kwe = TAILQ_LAST(&kq->ksynq_kwelist, ksynq_kwelist_head);
kq->ksynq_lastnum = (q_kwe->kwe_lockseq & PTHRW_COUNT_MASK);
} else {
kq->ksynq_firstnum = 0;
kq->ksynq_lastnum = 0;
}
kwq->kw_inqueue--;
curseq = kwe->kwe_lockseq & PTHRW_COUNT_MASK;
if (kwq->kw_inqueue == 0) {
kwq->kw_lowseq = 0;
kwq->kw_highseq = 0;
} else {
if (kwq->kw_lowseq == curseq)
kwq->kw_lowseq = find_nextlowseq(kwq);
if (kwq->kw_highseq == curseq)
kwq->kw_highseq = find_nexthighseq(kwq);
}
}
}
/* find the thread and removes from the queue */
ksyn_waitq_element_t
ksyn_queue_find_seq(ksyn_wait_queue_t kwq, ksyn_queue_t kq, uint32_t seq, int remove)
{
ksyn_waitq_element_t q_kwe, r_kwe;
/* TBD: bail out if higher seq is seen */
/* case where wrap in the tail of the queue exists */
TAILQ_FOREACH_SAFE(q_kwe, &kq->ksynq_kwelist, kwe_list, r_kwe) {
if ((q_kwe->kwe_lockseq & PTHRW_COUNT_MASK) == seq) {
if (remove != 0)
ksyn_queue_removeitem(kwq, kq, q_kwe);
return(q_kwe);
}
}
return(NULL);
}
/* find the thread at the target sequence (or a broadcast/prepost at or above) */
ksyn_waitq_element_t
ksyn_queue_find_cvpreposeq(ksyn_queue_t kq, uint32_t cgen)
{
ksyn_waitq_element_t q_kwe, r_kwe;
uint32_t lgen = (cgen & PTHRW_COUNT_MASK);
/* case where wrap in the tail of the queue exists */
TAILQ_FOREACH_SAFE(q_kwe, &kq->ksynq_kwelist, kwe_list, r_kwe) {
/* skip the lower entries */
if (is_seqlower((q_kwe->kwe_lockseq & PTHRW_COUNT_MASK), cgen) != 0)
continue;
switch (q_kwe->kwe_flags) {
case KWE_THREAD_INWAIT:
if ((q_kwe->kwe_lockseq & PTHRW_COUNT_MASK) != lgen)
break;
/* fall thru */
case KWE_THREAD_BROADCAST:
case KWE_THREAD_PREPOST:
return (q_kwe);
}
}
return(NULL);
}
/* look for a thread at lockseq, a */
ksyn_waitq_element_t
ksyn_queue_find_signalseq(__unused ksyn_wait_queue_t kwq, ksyn_queue_t kq, uint32_t uptoseq, uint32_t signalseq)
{
ksyn_waitq_element_t q_kwe, r_kwe, t_kwe = NULL;
/* case where wrap in the tail of the queue exists */
TAILQ_FOREACH_SAFE(q_kwe, &kq->ksynq_kwelist, kwe_list, r_kwe) {
switch (q_kwe->kwe_flags) {
case KWE_THREAD_PREPOST:
if (is_seqhigher((q_kwe->kwe_lockseq & PTHRW_COUNT_MASK), uptoseq))
return t_kwe;
/* fall thru */
case KWE_THREAD_BROADCAST:
/* match any prepost at our same uptoseq or any broadcast above */
if (is_seqlower((q_kwe->kwe_lockseq & PTHRW_COUNT_MASK), uptoseq))
continue;
return q_kwe;
case KWE_THREAD_INWAIT:
/*
* Match any (non-cancelled) thread at or below our upto sequence -
* but prefer an exact match to our signal sequence (if present) to
* keep exact matches happening.
*/
if (is_seqhigher((q_kwe->kwe_lockseq & PTHRW_COUNT_MASK), uptoseq))
return t_kwe;
if (q_kwe->kwe_kwqqueue == kwq) {
uthread_t ut = q_kwe->kwe_uth;
#ifndef __DARLING__
if ((ut->uu_flag & ( UT_CANCELDISABLE | UT_CANCEL | UT_CANCELED)) != UT_CANCEL) {
#else
if (1) {
#endif
/* if equal or higher than our signal sequence, return this one */
if (is_seqhigher_eq((q_kwe->kwe_lockseq & PTHRW_COUNT_MASK), signalseq))
return q_kwe;
/* otherwise, just remember this eligible thread and move on */
if (t_kwe == NULL)
t_kwe = q_kwe;
}
}
break;
default:
panic("ksyn_queue_find_signalseq(): unknow wait queue element type (%d)\n", q_kwe->kwe_flags);
break;
}
}
return t_kwe;
}
int
ksyn_queue_move_tofree(ksyn_wait_queue_t ckwq, ksyn_queue_t kq, uint32_t upto, ksyn_queue_t kfreeq, int all, int release)
{
ksyn_waitq_element_t kwe;
int count = 0;
uint32_t tseq = upto & PTHRW_COUNT_MASK;
#if _PSYNCH_TRACE_
uthread_t ut;
#endif /* _PSYNCH_TRACE_ */
ksyn_queue_init(kfreeq);
/* free all the entries, must be only fakes.. */
kwe = TAILQ_FIRST(&kq->ksynq_kwelist);
while (kwe != NULL) {
if ((all == 0) && (is_seqhigher((kwe->kwe_lockseq & PTHRW_COUNT_MASK), tseq) != 0))
break;
if (kwe->kwe_flags == KWE_THREAD_INWAIT) {
/*
* This scenario is typically noticed when the cvar is
* reinited and the new waiters are waiting. We can
* return them as spurious wait so the cvar state gets
* reset correctly.
*/
#if _PSYNCH_TRACE_
ut = (uthread_t)kwe->kwe_uth;
#endif /* _PSYNCH_TRACE_ */
/* skip canceled ones */
/* wake the rest */
ksyn_queue_removeitem(ckwq, kq, kwe);
/* set M bit to indicate to waking CV to retun Inc val */
kwe->kwe_psynchretval = PTHRW_INC | (PTH_RWS_CV_MBIT | PTH_RWL_MTX_WAIT);
kwe->kwe_kwqqueue = NULL;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVHBROAD | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xcafecaf3, (uint32_t)(thread_tid((struct thread *)(((struct uthread *)(kwe->kwe_uth))->uu_context.vc_thread))), kwe->kwe_psynchretval, 0);
#endif /* _PSYNCH_TRACE_ */
(void)ksyn_wakeup_thread(ckwq, kwe);
} else {
ksyn_queue_removeitem(ckwq, kq, kwe);
TAILQ_INSERT_TAIL(&kfreeq->ksynq_kwelist, kwe, kwe_list);
ckwq->kw_fakecount--;
count++;
}
kwe = TAILQ_FIRST(&kq->ksynq_kwelist);
}
if ((release != 0) && (count != 0)) {
kwe = TAILQ_FIRST(&kfreeq->ksynq_kwelist);
while (kwe != NULL) {
TAILQ_REMOVE(&kfreeq->ksynq_kwelist, kwe, kwe_list);
zfree(kwe_zone, kwe);
kwe = TAILQ_FIRST(&kfreeq->ksynq_kwelist);
}
}
return(count);
}
/*************************************************************************/
void
update_low_high(ksyn_wait_queue_t kwq, uint32_t lockseq)
{
if (kwq->kw_inqueue == 1) {
kwq->kw_lowseq = lockseq;
kwq->kw_highseq = lockseq;
} else {
if (is_seqlower(lockseq, kwq->kw_lowseq) != 0)
kwq->kw_lowseq = lockseq;
if (is_seqhigher(lockseq, kwq->kw_highseq) != 0)
kwq->kw_highseq = lockseq;
}
}
uint32_t
find_nextlowseq(ksyn_wait_queue_t kwq)
{
uint32_t numbers[KSYN_QUEUE_MAX];
int count = 0, i;
uint32_t lowest;
for(i = 0; i< KSYN_QUEUE_MAX; i++) {
if (kwq->kw_ksynqueues[i].ksynq_count != 0) {
numbers[count]= kwq->kw_ksynqueues[i].ksynq_firstnum;
count++;
}
}
if (count == 0)
return(0);
lowest = numbers[0];
if (count > 1) {
for (i = 1; i< count; i++) {
if(is_seqlower(numbers[i] , lowest) != 0)
lowest = numbers[count];
}
}
return(lowest);
}
uint32_t
find_nexthighseq(ksyn_wait_queue_t kwq)
{
uint32_t numbers[KSYN_QUEUE_MAX];
int count = 0, i;
uint32_t highest;
for(i = 0; i< KSYN_QUEUE_MAX; i++) {
if (kwq->kw_ksynqueues[i].ksynq_count != 0) {
numbers[count]= kwq->kw_ksynqueues[i].ksynq_lastnum;
count++;
}
}
if (count == 0)
return(0);
highest = numbers[0];
if (count > 1) {
for (i = 1; i< count; i++) {
if(is_seqhigher(numbers[i], highest) != 0)
highest = numbers[i];
}
}
return(highest);
}
int
is_seqlower(uint32_t x, uint32_t y)
{
if (x < y) {
if ((y-x) < (PTHRW_MAX_READERS/2))
return(1);
} else {
if ((x-y) > (PTHRW_MAX_READERS/2))
return(1);
}
return(0);
}
int
is_seqlower_eq(uint32_t x, uint32_t y)
{
if (x==y)
return(1);
else
return(is_seqlower(x,y));
}
int
is_seqhigher(uint32_t x, uint32_t y)
{
if (x > y) {
if ((x-y) < (PTHRW_MAX_READERS/2))
return(1);
} else {
if ((y-x) > (PTHRW_MAX_READERS/2))
return(1);
}
return(0);
}
int
is_seqhigher_eq(uint32_t x, uint32_t y)
{
if (x==y)
return(1);
else
return(is_seqhigher(x,y));
}
int
find_diff(uint32_t upto, uint32_t lowest)
{
uint32_t diff;
if (upto == lowest)
return(0);
#if 0
diff = diff_genseq(upto, lowest);
#else
if (is_seqlower(upto, lowest) != 0)
diff = diff_genseq(lowest, upto);
else
diff = diff_genseq(upto, lowest);
#endif
diff = (diff >> PTHRW_COUNT_SHIFT);
return(diff);
}
int
find_seq_till(ksyn_wait_queue_t kwq, uint32_t upto, uint32_t nwaiters, uint32_t *countp)
{
int i;
uint32_t count = 0;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_FSEQTILL | DBG_FUNC_START, 0, 0, upto, nwaiters, 0);
#endif /* _PSYNCH_TRACE_ */
for (i= 0; i< KSYN_QUEUE_MAX; i++) {
count += ksyn_queue_count_tolowest(&kwq->kw_ksynqueues[i], upto);
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_FSEQTILL | DBG_FUNC_NONE, 0, 1, i, count, 0);
#endif /* _PSYNCH_TRACE_ */
if (count >= nwaiters) {
break;
}
}
if (countp != NULL) {
*countp = count;
}
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_FSEQTILL | DBG_FUNC_END, 0, 0, count, nwaiters, 0);
#endif /* _PSYNCH_TRACE_ */
if (count == 0)
return(0);
else if (count >= nwaiters)
return(1);
else
return(0);
}
uint32_t
ksyn_queue_count_tolowest(ksyn_queue_t kq, uint32_t upto)
{
uint32_t i = 0;
ksyn_waitq_element_t kwe, newkwe;
uint32_t curval;
/* if nothing or the first num is greater than upto, return none */
if ((kq->ksynq_count == 0) || (is_seqhigher(kq->ksynq_firstnum, upto) != 0))
return(0);
if (upto == kq->ksynq_firstnum)
return(1);
TAILQ_FOREACH_SAFE(kwe, &kq->ksynq_kwelist, kwe_list, newkwe) {
curval = (kwe->kwe_lockseq & PTHRW_COUNT_MASK);
if (upto == curval) {
i++;
break;
} else if (is_seqhigher(curval, upto) != 0) {
break;
} else {
/* seq is lower */
i++;
}
}
return(i);
}
/* handles the cond broadcast of cvar and returns number of woken threads and bits for syscall return */
void
ksyn_handle_cvbroad(ksyn_wait_queue_t ckwq, uint32_t upto, uint32_t * updatep)
{
kern_return_t kret;
ksyn_queue_t kq;
ksyn_waitq_element_t kwe, newkwe;
uint32_t updatebits = 0;
struct ksyn_queue kfreeq;
uthread_t ut;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVHBROAD | DBG_FUNC_START, 0xcbcbcbc2, upto, 0, 0, 0);
#endif /* _PSYNCH_TRACE_ */
ksyn_queue_init(&kfreeq);
kq = &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER];
retry:
TAILQ_FOREACH_SAFE(kwe, &kq->ksynq_kwelist, kwe_list, newkwe) {
if (is_seqhigher((kwe->kwe_lockseq & PTHRW_COUNT_MASK), upto)) /* outside our range */
break;
/* now handle the one we found (inside the range) */
switch (kwe->kwe_flags) {
case KWE_THREAD_INWAIT:
ut = (uthread_t)kwe->kwe_uth;
#ifndef __DARLING__
/* skip canceled ones */
if (kwe->kwe_kwqqueue != ckwq ||
(ut->uu_flag & (UT_CANCELDISABLE | UT_CANCEL | UT_CANCELED)) == UT_CANCEL)
break;
#endif
/* wake the rest */
ksyn_queue_removeitem(ckwq, kq, kwe);
kwe->kwe_psynchretval = PTH_RWL_MTX_WAIT;
kwe->kwe_kwqqueue = NULL;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVHBROAD | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xcafecaf2, (uint32_t)(thread_tid((struct thread *)(((struct uthread *)(kwe->kwe_uth))->uu_context.vc_thread))), kwe->kwe_psynchretval, 0);
#endif /* _PSYNCH_TRACE_ */
kret = ksyn_wakeup_thread(ckwq, kwe);
#if __TESTPANICS__
if ((kret != KERN_SUCCESS) && (kret != KERN_NOT_WAITING))
panic("ksyn_wakeupreaders: panic waking up readers\n");
#endif /* __TESTPANICS__ */
updatebits += PTHRW_INC;
break;
case KWE_THREAD_BROADCAST:
case KWE_THREAD_PREPOST:
ksyn_queue_removeitem(ckwq, kq, kwe);
TAILQ_INSERT_TAIL(&kfreeq.ksynq_kwelist, kwe, kwe_list);
ckwq->kw_fakecount--;
break;
default:
panic("unknown kweflags\n");
break;
}
}
/* Need to enter a broadcast in the queue (if not already at L == S) */
if ((ckwq->kw_lword & PTHRW_COUNT_MASK) != (ckwq->kw_sword & PTHRW_COUNT_MASK)) {
newkwe = TAILQ_FIRST(&kfreeq.ksynq_kwelist);
if (newkwe == NULL) {
ksyn_wqunlock(ckwq);
newkwe = (ksyn_waitq_element_t) kzalloc(sizeof(struct ksyn_waitq_element), GFP_KERNEL); //zalloc(kwe_zone);
TAILQ_INSERT_TAIL(&kfreeq.ksynq_kwelist, newkwe, kwe_list);
ksyn_wqlock(ckwq);
goto retry;
}
TAILQ_REMOVE(&kfreeq.ksynq_kwelist, newkwe, kwe_list);
bzero(newkwe, sizeof(struct ksyn_waitq_element));
sema_init(&newkwe->linux_sem, 0);
newkwe->kwe_kwqqueue = ckwq;
newkwe->kwe_flags = KWE_THREAD_BROADCAST;
newkwe->kwe_lockseq = upto;
newkwe->kwe_count = 0;
newkwe->kwe_uth = NULL;
newkwe->kwe_psynchretval = 0;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVHBROAD | DBG_FUNC_NONE, (uint32_t)ckwq->kw_addr, 0xfeedfeed, upto, 0, 0);
#endif /* _PSYNCH_TRACE_ */
(void)ksyn_queue_insert(ckwq, &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER], upto, NULL, newkwe, SEQFIT);
ckwq->kw_fakecount++;
}
/* free up any remaining things stumbled across above */
kwe = TAILQ_FIRST(&kfreeq.ksynq_kwelist);
while (kwe != NULL) {
TAILQ_REMOVE(&kfreeq.ksynq_kwelist, kwe, kwe_list);
zfree(kwe_zone, kwe);
kwe = TAILQ_FIRST(&kfreeq.ksynq_kwelist);
}
if (updatep != NULL)
*updatep = updatebits;
#if _PSYNCH_TRACE_
__PTHREAD_TRACE_DEBUG(_PSYNCH_TRACE_CVHBROAD | DBG_FUNC_END, 0xeeeeeeed, updatebits, 0, 0, 0);
#endif /* _PSYNCH_TRACE_ */
}
void
ksyn_cvupdate_fixup(ksyn_wait_queue_t ckwq, uint32_t *updatep, ksyn_queue_t kfreeq, int release)
{
uint32_t updatebits = 0;
if (updatep != NULL)
updatebits = *updatep;
if ((ckwq->kw_lword & PTHRW_COUNT_MASK) == (ckwq->kw_sword & PTHRW_COUNT_MASK)) {
updatebits |= PTH_RWS_CV_CBIT;
if (ckwq->kw_inqueue != 0) {
/* FREE THE QUEUE */
ksyn_queue_move_tofree(ckwq, &ckwq->kw_ksynqueues[KSYN_QUEUE_WRITER], ckwq->kw_lword, kfreeq, 0, release);
#if __TESTPANICS__
if (ckwq->kw_inqueue != 0)
panic("ksyn_cvupdate_fixup: L == S, but entries in queue beyond S");
#endif /* __TESTPANICS__ */
}
ckwq->kw_lword = ckwq->kw_uword = ckwq->kw_sword = 0;
ckwq->kw_kflags |= KSYN_KWF_ZEROEDOUT;
} else if ((ckwq->kw_inqueue != 0) && (ckwq->kw_fakecount == ckwq->kw_inqueue)) {
/* only fake entries are present in the queue */
updatebits |= PTH_RWS_CV_PBIT;
}
if (updatep != NULL)
*updatep = updatebits;
}
#if 0
void
psynch_zoneinit(void)
{
kwq_zone = (zone_t)zinit(sizeof(struct ksyn_wait_queue), 8192 * sizeof(struct ksyn_wait_queue), 4096, "ksyn_waitqueue zone");
kwe_zone = (zone_t)zinit(sizeof(struct ksyn_waitq_element), 8192 * sizeof(struct ksyn_waitq_element), 4096, "ksyn_waitq_element zone");
}
#endif
Reference in New Issue View Git Blame Copy Permalink